GCC Code Coverage Report

Directory: ./
File: plugin/group_replication/libmysqlgcs/src/bindings/xcom/xcom/xcom_base.cc
Date: 2022-12-13 11:44:05
Exec Total Coverage
Lines: 2759 3495 78.9%
Branches: 1922 3808 50.5%
Line Branch Exec Source
1 /* Copyright (c) 2012, 2022, Oracle and/or its affiliates.
2
3 This program is free software; you can redistribute it and/or modify
4 it under the terms of the GNU General Public License, version 2.0,
5 as published by the Free Software Foundation.
6
7 This program is also distributed with certain software (including
8 but not limited to OpenSSL) that is licensed under separate terms,
9 as designated in a particular file or component or in included license
10 documentation. The authors of MySQL hereby grant you an additional
11 permission to link the program and your derivative works with the
12 separately licensed software that they have included with MySQL.
13
14 This program is distributed in the hope that it will be useful,
15 but WITHOUT ANY WARRANTY; without even the implied warranty of
16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 GNU General Public License, version 2.0, for more details.
18
19 You should have received a copy of the GNU General Public License
20 along with this program; if not, write to the Free Software
21 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
22
23 #include <assert.h>
24 #include <errno.h>
25 #ifndef __STDC_FORMAT_MACROS
26 #define __STDC_FORMAT_MACROS
27 #endif
28 #ifndef _WIN32
29 #include <inttypes.h>
30 #endif
31 #include <limits.h>
32 #include <signal.h>
33 #include <stdio.h>
34 #include <stdlib.h>
35 #include <string.h>
36 #include <sys/time.h>
37 #include <sys/types.h>
38 #ifdef _MSC_VER
39 #include <stdint.h>
40 #endif
41
42 #ifndef _WIN32
43 #include <poll.h>
44 #endif
45
46 /**
47 @file
48 xcom/xcom_base.c
49 The new version of xcom is a major rewrite to allow
50 transmission of multiple messages from several sources
51 simultaneously without collision. The interface to xcom is largely
52 intact, one notable change is that xcom will consider the message
53 delivered as soon as it has got a majority. Consequently, the VP
54 set will not necessarily show all nodes which will actually
55 receive the message.
56
57 OHKFIX Add wait for complete last known node set to mimic the old
58 semantics.
59
60
61 IMPORTANT: What xcom does and what it does not do:
62
63 xcom messages are received in the same order on all nodes.
64
65 xcom guarantees that if a message is delivered to one node, it will
66 eventually be seen on all other nodes as well.
67
68 xcom messages are available to a crashed node when it comes up
69 again if at least one node which knows the value of the message
70 has not crashed. The size of the message cache is configurable.
71
72 OHKFIX Add logging to disk to make messages durable across system
73 crash and to increase the number of messages which may be cached.
74
75 There is no guarantee whatsoever about the order of messages from
76 different nodes, not even the order of multiple messages from the
77 same node. It is up to the client to impose such an order by
78 waiting on a message before it sends the next.
79
80 xcom can notify the client that a message has timed out, and in
81 that case will try to cancel the message, but it cannot guarantee
82 that a message which has timed out will not be delivered.
83
84 xcom attaches a node set to each message as it is delivered to the
85 client. This node set reflects the current node set that xcom
86 believes is active, it does not mean that the message has been
87 delivered yet to all nodes in the set. Neither does it mean that
88 the message has not been delivered to the nodes not in the set.
89
90 A cache of Paxos state machines is central to the new design. The
91 purpose of the cache is both to store a window of messages, and to
92 decouple the different parts of xcom, like message proposal,
93 message delivery and execution, and recovery. The old cache was
94 limited to caching messages, and a single state machine ran the
95 combined VP and Paxos algorithm. This constrained xcom to deliver
96 only a single message at a time.
97
98 Each instance of the Paxos state machine implements the basic
99 Paxos protocol. Unlike the cache in the old system, it is not
100 cleared when a site is deleted. This removes some problems
101 related to message delivery during site deletion. The cache is a
102 classic fixed size LRU with a hash index.
103
104 Some extensions to the basic Paxos algorithm has been implemented:
105
106 A node has ownership to all synodes with its own node number. Only
107 a node with node number N can propose a value for synode {X N},
108 where X is the sequence number, and N is the node number. Other
109 nodes can only propose the special value no_op for synode {X N}.
110 The reason for this is to retain the leaderless Paxos algorithm,
111 but to avoid collisions between nodes which are competing for the
112 same synode number. With this scheme, each node has its own unique
113 number series during normal operation. The scheme has the
114 following implications:
115
116 1. If a node N has not already proposed a value for the synode {X N},
117 it may at any time send a LEARN message to the other nodes with
118 the reserved value no_op, without going through phase 1 and 2 of
119 Paxos. This is because the other nodes are constrained to propose
120 no_op for this synode, so the final outcome will always be no_op.
121 To avoid unnecessary message transmission, a node will try to
122 broadcast the no_op LEARN messages by piggybacking the information
123 on the messages of the basic Paxos protocol.
124
125 2. Other nodes which want to find the value of synode {X N} may do
126 so by trying to get the value no_op accepted by following the
127 basic Paxos algorithm. The result will be the actual value
128 proposed by node N if it has done so, otherwise no_op. This will
129 typically only be necessary when a node is down, and the other
130 nodes need to find the values from the missing node in order to be
131 able to continue execution.
132
133 Messages are delivered in order to the client, and the order is
134 determined by the sequence number and the node number, with the
135 sequence number as the most significant part.
136
137 The xcom network interface has been redesigned and is now
138 implemented directly on top of TCP, and has so far been completely
139 trouble free. We use poll() or select() to implement non-blocking
140 send and receive, but libev could equally well have been used.
141
142 Multicast is implemented on top of unicast as before, but the
143 implementation is prepared to use real multicast with relatively
144 minor changes.
145
146 The roles of proposer, acceptor/learner, and executor are now
147 directly mapped to unique task types which interact with the Paxos
148 state machines, whereas the previous implementation folded all the
149 roles into a single event driven state machine.
150
151 The following terminology will be used:
152
153 A node is an instance of the xcom thread. There is only one instance
154 of the xcom thread in the agent.
155 A client is the application which is using xcom to send messages.
156 A thread is a real OS thread.
157 A task is a logical process. It is implemented by coroutines and
158 an explicit stack.
159
160 The implementation of tasks and non-blocking socket operations is
161 isolated in task.h and task.c.
162
163 A node will open a tcp connection to each of the other nodes. This
164 connection is used for all communication initiated by the node,
165 and replies to messages will arrive on the connection on which it
166 was sent.
167
168 static int tcp_server(task_arg);
169
170 The tcp_server listens on the xcom port and starts an
171 acceptor_learner_task whenever a new connection is detected.
172
173 static int tcp_reaper_task(task_arg);
174
175 Closes tcp connection which have been unused for too long.
176
177 static int sender_task(task_arg);
178
179 The sender_task waits for tcp messages on its input queue and
180 sends it on the tcp socket. If the socket is closed for any
181 reason, the sender_task will reconnect the socket. There is one
182 sender_task for each socket. The sender task exists mainly to
183 simplify the logic in the other tasks, but it could have been
184 replaced with a coroutine which handles the connection logic after
185 having reserved the socket for its client task.
186
187 static int generator_task(task_arg);
188
189 The generator_task reads messages from the client queue and moves
190 them into the input queue of the proposer_task.
191
192 OHKFIX Use a tcp socket instead of the client queue. We can then
193 remove the generator_task and let the acceptor_learner_task do the
194 dispatching.
195
196 static int proposer_task(task_arg);
197
198 Assign a message number to an outgoing message and try to get it
199 accepted. There may be several proposer tasks on each node
200 working in parallel. If there are multiple proposer tasks, xcom can
201 not guarantee that the messages will be sent in the same order as
202 received from the client.
203
204 static int acceptor_learner_task(task_arg);
205
206 This is the server part of the xcom thread. There is one
207 acceptor_learner_task for each node in the system. The acceptor
208 learner_task reads messages from the socket, finds the correct
209 Paxos state machine, and dispatches to the correct message handler
210 with the state machine and message as arguments.
211
212 static int reply_handler_task(task_arg);
213
214 The reply_handler_task does the same job as the
215 acceptor_learner_task, but listens on the socket which the node
216 uses to send messages, so it will handle only replies on that
217 socket.
218
219 static int executor_task(task_arg);
220
221 The ececutor_task waits for a Paxos message to be accpeted. When
222 the message is accepted, it is delivered to the client,
223 unless it is a no-op. In either case, the executor_task steps to
224 the next message and repeats the wait. If it times out waiting for
225 a message, it will try to get a no-op accepted.
226
227 static int alive_task(task_arg);
228
229 Sends i-am-alive to other nodes if there has been no normal traffic
230 for a while. It also pings nodes which seem to be inactive.
231
232 static int detector_task(task_arg);
233
234 The detector_task periodically scans the set of connections from
235 other nodes and sees if there has been any activity. If there has
236 been no activity for some time, it will assume that the node is
237 dead, and send a view message to the client.
238
239
240 Reconfiguration:
241
242 The xcom reconfiguration process is essentially the one described in
243 "Reconfiguring a State Machine" by Lamport et al. as the R-alpha
244 algorithm.
245 We execute the reconfiguration command immediately, but the config is
246 only valid after a delay of alpha messages.
247 The parameter alpha is the same as
248 EVENT_HORIZON in this implementation. :/static.*too_far
249 All tcp messages from beyond the event horizon will be ignored.
250
251 */
252 #include "xcom/xcom_profile.h"
253
254 #ifndef XCOM_STANDALONE
255 #include "my_compiler.h"
256 #endif
257 #include "xcom/x_platform.h"
258
259 #ifndef _WIN32
260 #include <arpa/inet.h>
261 #include <net/if.h>
262 #include <sys/ioctl.h>
263 #include <sys/socket.h>
264 #ifndef __linux__
265 #include <sys/sockio.h>
266 #endif
267 #endif
268
269 #if defined(_WIN32)
270 #include <windows.h>
271 #endif
272
273 #include <memory>
274
275 #include "xcom/app_data.h"
276 #include "xcom/get_synode_app_data.h"
277 #include "xcom/node_no.h"
278 #include "xcom/server_struct.h"
279 #include "xcom/simset.h"
280 #include "xcom/site_struct.h"
281 #include "xcom/task.h"
282 #include "xcom/task_net.h"
283 #include "xcom/task_os.h"
284 #include "xcom/xcom_base.h"
285 #include "xcom/xcom_common.h"
286 #include "xcom/xcom_detector.h"
287 #include "xcom/xcom_transport.h"
288 #include "xcom/xdr_utils.h"
289 #include "xdr_gen/xcom_vp.h"
290
291 #include "xcom/bitset.h"
292 #include "xcom/leader_info_data.h"
293 #include "xcom/node_list.h"
294 #include "xcom/node_set.h"
295 #include "xcom/pax_msg.h"
296 #include "xcom/site_def.h"
297 #include "xcom/sock_probe.h"
298 #include "xcom/synode_no.h"
299 #include "xcom/task_debug.h"
300 #include "xcom/task_net.h"
301 #include "xcom/xcom_cache.h"
302 #include "xcom/xcom_cfg.h"
303 #include "xcom/xcom_interface.h"
304 #include "xcom/xcom_memory.h"
305 #include "xcom/xcom_msg_queue.h"
306 #include "xcom/xcom_recover.h"
307 #include "xcom/xcom_statistics.h"
308 #include "xcom/xcom_vp_str.h"
309
310 #include "xcom/network/xcom_network_provider.h"
311
312 #ifndef XCOM_WITHOUT_OPENSSL
313 #ifdef _WIN32
314 /* In OpenSSL before 1.1.0, we need this first. */
315 #include <winsock2.h>
316 #endif /* _WIN32 */
317
318 #include <openssl/ssl.h>
319
320 #endif
321
322 #include <queue>
323 #include <tuple>
324
325 /* Defines and constants */
326
327 #define SYS_STRERROR_SIZE 512
328
329 /* Avoid printing the warning of protocol version mismatch too often */
330 #define PROTOVERSION_WARNING_TIMEOUT 600.0 /** Every 10 minutes */
331 static double protoversion_warning_time =
332 0.0; /** Timestamp of previous protoversion warning */
333
334 /* Skip prepare for first ballot */
335 #ifdef ALWAYS_THREEPHASE
336 int const threephase = 1;
337 #else
338 int const threephase = 0;
339 #endif
340
341 #include "xcom/retry.h"
342
343 #ifdef NODE_0_IS_ARBITRATOR
344 int ARBITRATOR_HACK = 1;
345 #else
346 int ARBITRATOR_HACK = 0;
347 #endif
348
349 static int const no_duplicate_payload = 1;
350
351 /* Use buffered read when reading messages from the network */
352 static int use_buffered_read = 1;
353
354 /* Used to handle OOM errors */
355 int oom_abort = 0;
356
357 /* Forward declarations */
358 long xcom_unique_long(void);
359 static int64_t socket_write(
360 connection_descriptor *wfd, void *_buf, uint32_t n,
361 connnection_write_method write_function = con_write);
362
363 static double wakeup_delay(double old);
364 static void note_snapshot(node_no node);
365
366 /* Task types */
367 static int proposer_task(task_arg arg);
368 static int executor_task(task_arg arg);
369 static int sweeper_task(task_arg arg);
370 extern int alive_task(task_arg arg);
371 extern int cache_manager_task(task_arg arg);
372 extern int detector_task(task_arg arg);
373
374 static int finished(pax_machine *p);
375 static int accepted(pax_machine *p);
376 static int started(pax_machine *p);
377 static synode_no first_free_synode_local(synode_no msgno);
378 static void free_forced_config_site_def();
379 static void activate_sweeper();
380 static void force_pax_machine(pax_machine *p, int enforcer);
381 static void propose_noop_2p(synode_no find, pax_machine *p);
382 static void handle_need_snapshot(linkage *reply_queue, pax_msg *pm);
383 static void handle_skip(site_def const *site, pax_machine *p, pax_msg *m);
384 static void paxos_fsm(pax_machine *paxos, site_def const *site,
385 paxos_event event, pax_msg *mess);
386 static inline bool is_leader(site_def *site);
387 int is_active_leader(node_no x, site_def *site);
388 static msg_handler *primary_dispatch_table();
389 static msg_handler *secondary_dispatch_table();
390 static void recompute_node_sets(site_def const *old_site, site_def *new_site);
391 void recompute_timestamps(detector_state const old_timestamp,
392 node_list const *old_nodes,
393 detector_state new_timestamp,
394 node_list const *new_nodes);
395 void analyze_leaders(site_def *site);
396 static inline int ignore_message(synode_no x, site_def *site, char const *dbg);
397
398 /* Global variables */
399
400 int xcom_shutdown = 0; /* Xcom_Shutdown flag */
401 synode_no executed_msg; /* The message we are waiting to execute */
402 synode_no max_synode; /* Max message number seen so far */
403 task_env *boot = nullptr;
404 task_env *detector = nullptr;
405 task_env *killer = nullptr;
406 task_env *net_boot = nullptr;
407 task_env *net_recover = nullptr;
408 void *xcom_thread_input = nullptr;
409
410 long xcom_debug_mask =
411 /* D_DETECT | */ D_FSM /* | D_FILEOP | D_CONS | D_BASE */ | D_TRANSPORT;
412 long xcom_dbg_stack[DBG_STACK_SIZE];
413 int xcom_dbg_stack_top = 0;
414
415 static void init_proposers();
416 void initialize_lsn(uint64_t n);
417
418 1215 void init_base_vars() {
419 1215 xcom_shutdown = 0; /* Xcom_Shutdown flag */
420 1215 executed_msg = null_synode; /* The message we are waiting to execute */
421 1215 max_synode = null_synode; /* Max message number seen so far */
422 1215 boot = nullptr;
423 1215 detector = nullptr;
424 1215 killer = nullptr;
425 1215 net_boot = nullptr;
426 1215 net_recover = nullptr;
427 1215 xcom_thread_input = nullptr;
428 1215 }
429
430 static task_env *executor = nullptr;
431 static task_env *sweeper = nullptr;
432 static task_env *retry = nullptr;
433 static task_env *proposer[PROPOSERS];
434 static task_env *alive_t = nullptr;
435 static task_env *cache_task = nullptr;
436
437 static uint32_t my_id = 0; /* Unique id of this instance */
438 653102 uint32_t get_my_xcom_id() { return my_id; }
439 static synode_no current_message; /* Current message number */
440 static synode_no
441 last_config_modification_id; /*Last configuration change proposal*/
442 static uint64_t lsn = 0; /* Current log sequence number */
443
444 58895 synode_no get_current_message() { return current_message; }
445
446 static channel prop_input_queue; /* Proposer task input queue */
447
448 enum class synode_allocation_type { todo = 0, local, remote, global };
449 enum class synode_reservation_status : int {
450 number_ok,
451 no_nodes,
452 delivery_timeout
453 };
454
455 #if 0
456 static char const *synode_allocation_type_to_str(synode_allocation_type x) {
457 switch (x) {
458 case synode_allocation_type::todo:
459 return "todo";
460 case synode_allocation_type::local:
461 return "local";
462 case synode_allocation_type::remote:
463 return "remote";
464 case synode_allocation_type::global:
465 return "global";
466 default:
467 return "";
468 }
469 }
470 #endif
471
472 // A pool of synode numbers implemented as a queue
473 struct synode_pool {
474 std::queue<std::pair<synode_no, synode_allocation_type>> data;
475 linkage queue;
476
477 1539 synode_pool() { link_init(&queue, TYPE_HASH("task_env")); }
478
479 402 void put(synode_no synode, synode_allocation_type allocation) {
480
1/2
✓ Branch 0 taken 402 times.
✗ Branch 1 not taken.
 402 data.push({synode, allocation});
481 402 task_wakeup(&queue);
482 402 }
483
484 402 auto get() {
485 402 auto retval = data.front();
486 402 data.pop();
487 402 return retval;
488 }
489
490 97764 bool empty() { return data.empty(); }
491 };
492
493 synode_pool synode_number_pool;
494
495 extern int client_boot_done;
496 extern int netboot_ok;
497
498 static linkage exec_wait = {
499 0, &exec_wait, &exec_wait}; /* Executor will wake up tasks sleeping here */
500
501 linkage detector_wait = {0, &detector_wait,
502 &detector_wait}; /* Detector sleeps here */
503
504 static struct {
505 int n;
506 unsigned long id[MAX_DEAD];
507 } dead_sites;
508
509 2523272 synode_no get_max_synode() { return max_synode; }
510
511 1693002 static bool_t is_latest_config(site_def const *const config) {
512 1693002 site_def const *const latest_config = get_site_def();
513
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1693002 times.
 1693002 assert(latest_config != nullptr);
514 1693002 return config == latest_config;
515 }
516
517 /**
518 * Get the first pending configuration that reconfigures the event horizon.
519 *
520 * Retrieve the first pending site_def, i.e. with the smallest start synod that
521 * is greater than executed_msg, that reconfigures the event horizon.
522 */
523 1686884 static site_def const *first_event_horizon_reconfig() {
524 1686884 site_def const *active_config = find_site_def(executed_msg);
525 1686884 xcom_event_horizon active_event_horizon = active_config->event_horizon;
526 1686884 site_def const *first_event_horizon_reconfig = nullptr;
527 1686884 site_def const *next_config = nullptr;
528 1686884 for (next_config = find_next_site_def(active_config->start);
529
4/4
✓ Branch 0 taken 495188 times.
✓ Branch 1 taken 1686074 times.
✓ Branch 2 taken 494378 times.
✓ Branch 3 taken 810 times.
 2181262 next_config != nullptr && first_event_horizon_reconfig == nullptr;
530 494378 next_config = find_next_site_def(next_config->start)) {
531
2/2
✓ Branch 0 taken 2687 times.
✓ Branch 1 taken 491691 times.
 494378 if (active_event_horizon != next_config->event_horizon) {
532 2687 first_event_horizon_reconfig = next_config;
533 }
534 }
535 1686884 return first_event_horizon_reconfig;
536 }
537
538 /**
539 * Get the latest pending configuration that reconfigures the event horizon.
540 *
541 * Retrieve the last pending site_def, i.e. with the greatest start synod that
542 * is greater than executed_msg, that reconfigures the event horizon.
543 */
544 6118 static site_def const *latest_event_horizon_reconfig() {
545 6118 site_def const *active_config = find_site_def(executed_msg);
546 6118 xcom_event_horizon previous_event_horizon = active_config->event_horizon;
547 6118 site_def const *last_event_horizon_reconfig = nullptr;
548 6118 site_def const *next_config = nullptr;
549 6118 for (next_config = find_next_site_def(active_config->start);
550
2/2
✓ Branch 0 taken 163 times.
✓ Branch 1 taken 6118 times.
 6281 next_config != nullptr;
551 163 next_config = find_next_site_def(next_config->start)) {
552
2/2
✓ Branch 0 taken 11 times.
✓ Branch 1 taken 152 times.
 163 if (previous_event_horizon != next_config->event_horizon) {
553 11 previous_event_horizon = next_config->event_horizon;
554 11 last_event_horizon_reconfig = next_config;
555 }
556 }
557 6118 return last_event_horizon_reconfig;
558 }
559
560 /**
561 * Add the event horizon to the given base synod s.
562 *
563 * We are assuming right now that this function is used solely in the context of
564 * "we have received a reconfiguration command at synod s, when should it be
565 * scheduled to take effect?"
566 * The result of this function is *when* it should take effect.
567 *
568 * Common case: there are no configurations pending, or if there are, none of
569 * them reconfigure the event horizon. The common case result is:
570 *
571 * s + event_horizon(active_config) + 1
572 *
573 *
574 * If an event horizon reconfiguration R is pending, it means that the command C
575 * proposed for synod s is concurrent with R, i.e., s falls in the interval
576 * ]proposed(R), start(R)[.
577 *
578 * In this situation we apply the command C proposed for synod s *after* taking
579 * into account R's event horizon.
580 *
581 * This means that the result is:
582 *
583 * start(R) + event_horizon(R) + 1
584 */
585 #ifdef PERMISSIVE_EH_ACTIVE_CONFIG
586 /* purecov: begin deadcode */
587 static synode_no add_default_event_horizon(synode_no s) {
588 s.msgno += EVENT_HORIZON_MIN + 1;
589 return s;
590 }
591 /* purecov: end */
592 #endif
593
594 6118 static synode_no add_event_horizon(synode_no s) {
595 6118 site_def const *active_config = find_site_def(executed_msg);
596
1/2
✓ Branch 0 taken 6118 times.
✗ Branch 1 not taken.
 6118 if (active_config) {
597 6118 site_def const *pending_config = latest_event_horizon_reconfig();
598 6118 bool_t const no_event_horizon_reconfig_pending =
599 6118 (pending_config == nullptr);
600
6/6
✓ Branch 0 taken 130 times.
✓ Branch 1 taken 5988 times.
✓ Branch 2 taken 122 times.
✓ Branch 3 taken 8 times.
✓ Branch 4 taken 6110 times.
✓ Branch 5 taken 8 times.
 6118 if (is_latest_config(active_config) || no_event_horizon_reconfig_pending) {
601 6110 s.msgno = s.msgno + active_config->event_horizon + 1;
602 } else {
603 8 s.msgno = pending_config->start.msgno + pending_config->event_horizon + 1;
604 }
605 6118 return s;
606 } else { /* This is initial boot or recovery, we have no config */
607 #ifdef PERMISSIVE_EH_ACTIVE_CONFIG
608 return add_default_event_horizon(s);
609 #else
610 /* We should always have an active config */
611 /* purecov: begin deadcode */
612 ✗ assert(active_config != nullptr);
613 ✗ return null_synode;
614 /* purecov: end */
615 #endif
616 }
617 }
618
619 /**
620 Set node group
621 */
622 12399 void set_group(uint32_t id) {
623 IFDBG(D_NONE, FN; STRLIT("changing group id of global variables ");
624 NDBG((unsigned long)id, lu););
625 /* set_group_id(id); */
626 12399 current_message.group_id = id;
627 12399 executed_msg.group_id = id;
628 12399 max_synode.group_id = id;
629 12399 }
630
631 2378 static void bury_site(uint32_t id) {
632
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2378 times.
 2378 if (id != 0) {
633 ✗ dead_sites.id[dead_sites.n % MAX_DEAD] = id;
634 ✗ dead_sites.n = (dead_sites.n + 1) % MAX_DEAD;
635 }
636 2378 }
637
638 547066 static bool_t is_dead_site(uint32_t id) {
639 547066 int i = 0;
640
1/2
✓ Branch 0 taken 547066 times.
✗ Branch 1 not taken.
 547066 for (i = 0; i < MAX_DEAD; i++) {
641
2/2
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 547065 times.
 547066 if (dead_sites.id[i] == id)
642 1 return TRUE;
643
1/2
✓ Branch 0 taken 547065 times.
✗ Branch 1 not taken.
 547065 else if (dead_sites.id[i] == 0)
644 547065 return FALSE;
645 }
646 ✗ return FALSE;
647 }
648
649 extern node_set *init_node_set(node_set *set, u_int n);
650 extern node_set *alloc_node_set(node_set *set, u_int n);
651
652 #if 0
653 /* Find our previous message number. */
654 static synode_no decr_msgno(synode_no msgno)
655 {
656 synode_no ret = msgno;
657 ret.msgno--;
658 ret.node = get_nodeno(find_site_def(ret)); /* In case site and node number has changed */
659 return ret;
660 }
661 #endif
662
663 /* Find our next message number. */
664 329620 static synode_no incr_msgno(synode_no msgno) {
665 329620 synode_no ret = msgno;
666 329620 ret.msgno++;
667 329620 ret.node = get_nodeno(
668 find_site_def(ret)); /* In case site and node number has changed */
669 329620 return ret;
670 }
671
672 1905925 synode_no incr_synode(synode_no synode) {
673 1905925 synode_no ret = synode;
674 1905925 ret.node++;
675
2/2
✓ Branch 0 taken 963201 times.
✓ Branch 1 taken 942724 times.
 1905925 if (ret.node >= get_maxnodes(find_site_def(synode))) {
676 963201 ret.node = 0;
677 963201 ret.msgno++;
678 }
679 /* IFDBG(D_NONE, FN; SYCEXP(synode); SYCEXP(ret)); */
680 1905925 return ret; /* Change this if we change message number type */
681 }
682
683 #if 0
684 synode_no decr_synode(synode_no synode) {
685 synode_no ret = synode;
686 if (ret.node == 0) {
687 ret.msgno--;
688 ret.node = get_maxnodes(find_site_def(ret));
689 }
690 ret.node--;
691 return ret; /* Change this if we change message number type */
692 }
693 #endif
694
695 248159 static void skip_value(pax_msg *p) {
696 IFDBG(D_NONE, FN; SYCEXP(p->synode));
697 248159 p->op = learn_op;
698 248159 p->msg_type = no_op;
699 248159 }
700
701 /* Utilities and debug */
702
703 #ifndef _WIN32
704 /* Ignore this signal */
705 2385 static int ignoresig(int signum) {
706 struct sigaction act;
707 struct sigaction oldact;
708
709 2385 memset(&act, 0, sizeof(act));
710 2385 act.sa_handler = SIG_IGN;
711 2385 memset(&oldact, 0, sizeof(oldact));
712
713 2385 return sigaction(signum, &act, &oldact);
714 }
715 #else
716 #define SIGPIPE 0
717 static int ignoresig(int signum) { return 0; }
718 #endif
719
720 686372 static int recently_active(pax_machine *p) {
721 IFDBG(D_NONE, FN; SYCEXP(p->synode); STRLIT(" op "); PTREXP(p);
722 STRLIT(p->learner.msg ? pax_op_to_str(p->learner.msg->op) : "NULL");
723 NDBG(p->last_modified, f); NDBG(task_now(), f));
724
2/2
✓ Branch 0 taken 235962 times.
✓ Branch 1 taken 450410 times.
 922334 return p->last_modified != 0.0 &&
725
2/2
✓ Branch 0 taken 218835 times.
✓ Branch 1 taken 17127 times.
 922334 (p->last_modified + BUILD_TIMEOUT + median_time()) > task_now();
726 }
727
728 3135743 static inline int finished(pax_machine *p) {
729 IFDBG(D_NONE, FN; SYCEXP(p->synode); STRLIT(" op "); PTREXP(p);
730 STRLIT(p->learner.msg ? pax_op_to_str(p->learner.msg->op) : "NULL"););
731
4/4
✓ Branch 0 taken 947576 times.
✓ Branch 1 taken 2188167 times.
✓ Branch 2 taken 36833 times.
✓ Branch 3 taken 910743 times.
 3172576 return p->learner.msg && (p->learner.msg->op == learn_op ||
732
1/2
✓ Branch 0 taken 36833 times.
✗ Branch 1 not taken.
 3172576 p->learner.msg->op == tiny_learn_op);
733 }
734
735 54160 int pm_finished(pax_machine *p) { return finished(p); }
736
737 161233 static inline int accepted(pax_machine *p) {
738 IFDBG(D_NONE, FN; SYCEXP(p->synode); STRLIT(" op "); PTREXP(p);
739 STRLIT(p->acceptor.msg ? pax_op_to_str(p->acceptor.msg->op) : "NULL"););
740
3/4
✓ Branch 0 taken 16625 times.
✓ Branch 1 taken 144608 times.
✓ Branch 2 taken 16625 times.
✗ Branch 3 not taken.
 161233 return p->acceptor.msg && p->acceptor.msg->op != initial_op;
741 }
742
743 14961 static inline int accepted_noop(pax_machine *p) {
744 IFDBG(D_NONE, FN; SYCEXP(p->synode); STRLIT(" op "); PTREXP(p);
745 STRLIT(p->acceptor.msg ? pax_op_to_str(p->acceptor.msg->op) : "NULL"););
746
4/4
✓ Branch 0 taken 260 times.
✓ Branch 1 taken 14701 times.
✓ Branch 2 taken 199 times.
✓ Branch 3 taken 61 times.
 14961 return accepted(p) && p->acceptor.msg->msg_type == no_op;
747 }
748
749 18150 static inline int noop_match(pax_machine *p, pax_msg *pm) {
750
4/4
✓ Branch 0 taken 14961 times.
✓ Branch 1 taken 3189 times.
✓ Branch 2 taken 199 times.
✓ Branch 3 taken 14762 times.
 18150 return pm->msg_type == no_op && accepted_noop(p);
751 }
752
753 114176 static inline int started(pax_machine *p) {
754
2/2
✓ Branch 0 taken 114148 times.
✓ Branch 1 taken 28 times.
 114176 return p->op != initial_op || (p->acceptor.promise.cnt > 0) ||
755
5/6
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 114147 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 times.
✓ Branch 4 taken 100727 times.
✓ Branch 5 taken 13420 times.
 228295 (p->proposer.msg && (p->proposer.msg->op != initial_op)) ||
756
2/4
✓ Branch 0 taken 114176 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 100727 times.
 342499 accepted(p) || finished(p);
757 }
758
759 11514 void set_last_received_config(synode_no received_config_change) {
760 11514 last_config_modification_id = received_config_change;
761 11514 }
762
763 /* Definition of majority */
764 381335 static inline node_no max_check(site_def const *site) {
765 #ifdef MAXACCEPT
766 return MIN(get_maxnodes(site), MAXACCEPT);
767 #else
768 381335 return get_maxnodes(site);
769 #endif
770 }
771
772 static site_def *forced_config = nullptr;
773 59450 static int is_forcing_node(pax_machine const *p) { return p->enforcer; }
774 static int wait_forced_config = 0;
775
776 /* Definition of majority */
777 381335 static inline int majority(bit_set const *nodeset, site_def const *s, int all,
778 int delay [[maybe_unused]], int force) {
779 381335 node_no ok = 0;
780 381335 node_no i = 0;
781 381335 int retval = 0;
782 #ifdef WAIT_FOR_ALL_FIRST
783 double sec = task_now();
784 #endif
785 381335 node_no max = max_check(s);
786
787 /* IFDBG(D_NONE, FN; NDBG(max,lu); NDBG(all,d); NDBG(delay,d); NDBG(force,d));
788 */
789
790 /* Count nodes that has answered */
791
2/2
✓ Branch 0 taken 887944 times.
✓ Branch 1 taken 381335 times.
 1269279 for (i = 0; i < max; i++) {
792
2/2
✓ Branch 0 taken 547553 times.
✓ Branch 1 taken 340391 times.
 887944 if (BIT_ISSET(i, nodeset)) {
793 547553 ok++;
794 }
795 #ifdef WAIT_FOR_ALL_FIRST
796 else {
797 if (all) return 0; /* Delay until all nodes have answered */
798 if (delay && !may_be_dead(s->detected, i, sec)) {
799 return 0; /* Delay until all live nodes have answered */
800 }
801 }
802 #endif
803 }
804
805 /* If we are forcing messages, attempt to ensure consistency by
806 requiring all remaining nodes to agree. Forced_config points to
807 the config that should be used as acceptors in this
808 case. Another possibility is to use the original config and
809 count the number of live nodes, but since the force flag is
810 being used only to force a new config, it seems safer to use
811 the new config and no time-dependent info. Note that we are
812 counting the answers based on the normal config, but use the
813 number of nodes from forced_config. This is safe, since we can
814 assume that the nodes that are not in forced_config will never
815 answer. */
816
817
2/2
✓ Branch 0 taken 6281 times.
✓ Branch 1 taken 375054 times.
 381335 if (force) {
818 IFDBG(D_NONE, FN; STRLIT("force majority"); NDBG(ok, u); NDBG(max, u);
819 NDBG(get_maxnodes(forced_config), u));
820 6281 return ok == get_maxnodes(forced_config);
821 } else {
822 /* Have now seen answer from all live nodes */
823 #ifdef NODE_0_IS_ARBITRATOR
824 retval = all ? ok == max
825 : ok > max / 2 ||
826 (ARBITRATOR_HACK && (get_nodeno(s) == 0) && (2 == max));
827 #else
828
4/8
✗ Branch 0 not taken.
✓ Branch 1 taken 375054 times.
✓ Branch 2 taken 177898 times.
✓ Branch 3 taken 197156 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 177898 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
 375054 retval = all ? ok == max : ok > max / 2 || (ARBITRATOR_HACK && (2 == max));
829 #endif
830 /* IFDBG(D_NONE, FN; NDBG(max,lu); NDBG(all,d); NDBG(delay,d);
831 * NDBG(retval,d)); */
832 375054 return retval;
833 }
834 }
835
836 #define IS_CONS_ALL(p) \
837 ((p)->proposer.msg->a ? (p)->proposer.msg->a->consensus == cons_all : 0)
838
839 /* See if a majority of acceptors have answered our prepare */
840 54400 static int prep_majority(site_def const *site, pax_machine const *p) {
841 54400 int ok = 0;
842
843
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 54400 times.
 54400 assert(p);
844
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 54400 times.
 54400 assert(p->proposer.prep_nodeset);
845
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 54400 times.
 54400 assert(p->proposer.msg);
846 /* IFDBG(D_NONE, FN; BALCEXP(p->proposer.bal)); */
847
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3225 times.
 54400 ok = majority(p->proposer.prep_nodeset, site, IS_CONS_ALL(p),
848
2/2
✓ Branch 0 taken 3225 times.
✓ Branch 1 taken 51175 times.
 54400 p->proposer.bal.cnt <= 1,
849
4/4
✓ Branch 0 taken 54262 times.
✓ Branch 1 taken 138 times.
✓ Branch 2 taken 2977 times.
✓ Branch 3 taken 51285 times.
 54400 p->proposer.msg->force_delivery || p->force_delivery);
850 54400 return ok;
851 }
852
853 /* See if a majority of acceptors have answered our propose */
854 326935 static int prop_majority(site_def const *site, pax_machine const *p) {
855 326935 int ok = 0;
856
857
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 326935 times.
 326935 assert(p);
858
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 326935 times.
 326935 assert(p->proposer.prop_nodeset);
859
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 326935 times.
 326935 assert(p->proposer.msg);
860 /* IFDBG(D_NONE, FN; BALCEXP(p->proposer.bal)); */
861
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 290928 times.
 326935 ok = majority(p->proposer.prop_nodeset, site, IS_CONS_ALL(p),
862
2/2
✓ Branch 0 taken 290928 times.
✓ Branch 1 taken 36007 times.
 326935 p->proposer.bal.cnt <= 1,
863
4/4
✓ Branch 0 taken 326771 times.
✓ Branch 1 taken 164 times.
✓ Branch 2 taken 3002 times.
✓ Branch 3 taken 323769 times.
 326935 p->proposer.msg->force_delivery || p->force_delivery);
864 326935 return ok;
865 }
866
867 /* Xcom thread */
868
869 static site_def *executor_site = nullptr;
870
871 1318 site_def const *get_executor_site() { return executor_site; }
872 58913 site_def *get_executor_site_rw() { return executor_site; }
873
874 static site_def *proposer_site = nullptr;
875
876 ✗ site_def const *get_proposer_site() { return proposer_site; }
877
878 /* delivered_msg may point to a no_op message, which will not actually be
879 * delivered */
880 static synode_no delivered_msg = NULL_SYNODE;
881
882 1554295 synode_no get_delivered_msg() { return delivered_msg; }
883
884 /* last_delivered_msg is the last synode we actually delivered */
885 static synode_no last_delivered_msg = NULL_SYNODE;
886 1318 synode_no get_last_delivered_msg() { return last_delivered_msg; }
887
888 3423 void init_xcom_base() {
889 IFDBG(D_NONE, FN);
890 3423 xcom_shutdown = 0;
891 3423 current_message = null_synode;
892 3423 executed_msg = null_synode;
893 3423 delivered_msg = null_synode;
894 3423 last_delivered_msg = null_synode;
895 3423 max_synode = null_synode;
896 3423 client_boot_done = 0;
897 3423 netboot_ok = 0;
898
899 3423 xcom_recover_init();
900 3423 my_id = new_id();
901 3423 push_site_def(nullptr);
902 /* update_servers(NULL); */
903 3423 xcom_cache_var_init();
904 3423 median_filter_init();
905 3423 link_init(&exec_wait, TYPE_HASH("task_env"));
906 3423 link_init(&detector_wait, TYPE_HASH("task_env"));
907 3423 link_init(&connect_wait, TYPE_HASH("task_env"));
908 3423 executor_site = nullptr;
909 3423 proposer_site = nullptr;
910
911 /** Reset lsn */
912 3423 initialize_lsn(0);
913 IFDBG(D_NONE, FN);
914 3423 }
915
916 3593 static void init_tasks() {
917 IFDBG(D_NONE, FN);
918 3593 set_task(&boot, nullptr);
919 3593 set_task(&net_boot, nullptr);
920 3593 set_task(&net_recover, nullptr);
921 3593 set_task(&killer, nullptr);
922 3593 set_task(&executor, nullptr);
923 3593 set_task(&retry, nullptr);
924 3593 set_task(&detector, nullptr);
925 3593 init_proposers();
926 3593 set_task(&alive_t, nullptr);
927 3593 set_task(&sweeper, nullptr);
928 3593 set_task(&cache_task, nullptr);
929 IFDBG(D_NONE, FN);
930 3593 }
931
932 /* Initialize the xcom thread */
933 1215 void xcom_thread_init() {
934 #ifndef NO_SIGPIPE
935 1215 signal(SIGPIPE, SIG_IGN);
936 #endif
937 1215 init_base_vars();
938 1215 init_site_vars();
939 1215 init_crc32c();
940 1215 xcom_srand48((long int)task_now());
941
942 1215 init_xcom_base();
943 1215 init_tasks();
944
945 /* Initialize input queue */
946 1215 channel_init(&prop_input_queue, TYPE_HASH("msg_link"));
947 1215 init_link_list();
948 1215 task_sys_init();
949
950 1215 init_cache();
951 1215 }
952
953 /* Empty the proposer input queue */
954 8016 static void empty_prop_input_queue() {
955 8016 empty_msg_channel(&prop_input_queue);
956 IFDBG(D_NONE, FN; STRLIT("prop_input_queue empty"));
957 8016 }
958
959 8016 static void empty_synode_number_pool() {
960
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 8016 times.
 8016 while (!synode_number_pool.data.empty()) {
961 ✗ synode_number_pool.data.pop();
962 }
963 8016 }
964
965 /* De-initialize the xcom thread */
966 2378 void xcom_thread_deinit() {
967 IFDBG(D_BUG, FN; STRLIT("Empty proposer input queue"));
968 2378 empty_prop_input_queue();
969 IFDBG(D_BUG, FN; STRLIT("Empty synode number pool"));
970 2378 empty_synode_number_pool();
971 IFDBG(D_BUG, FN; STRLIT("Empty link free list"));
972 2378 empty_link_free_list();
973 IFDBG(D_BUG, FN; STRLIT("De-initialize cache"));
974 2378 deinit_cache();
975 2378 garbage_collect_servers();
976 IFDBG(D_BUG, FN; STRLIT("De-initialize network cache"));
977 2378 deinit_network_cache();
978 IFDBG(D_BUG, FN; STRLIT("De-initialize xcom_interface"));
979 2378 deinit_xcom_interface();
980 2378 }
981
982 #define PROP_ITER \
983 int i; \
984 for (i = 0; i < PROPOSERS; i++)
985
986 5801 static void init_proposers() {
987
2/2
✓ Branch 0 taken 58010 times.
✓ Branch 1 taken 5801 times.
 63811 PROP_ITER { set_task(&proposer[i], nullptr); }
988 5801 }
989
990 2215 static void create_proposers() {
991
2/2
✓ Branch 0 taken 22150 times.
✓ Branch 1 taken 2215 times.
 24365 PROP_ITER {
992 22150 set_task(&proposer[i], task_new(proposer_task, int_arg(i), "proposer_task",
993 XCOM_THREAD_DEBUG));
994 }
995 2215 }
996
997 static synode_no *proposer_synodes[PROPOSERS];
998
999 44230 static void add_proposer_synode(int i, synode_no *syn_ptr) {
1000
2/4
✓ Branch 0 taken 44230 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 44230 times.
✗ Branch 3 not taken.
 44230 if (i >= 0 && i < PROPOSERS) {
1001 44230 proposer_synodes[i] = syn_ptr;
1002 }
1003 44230 }
1004
1005 22080 static void remove_proposer_synode(int i) { add_proposer_synode(i, nullptr); }
1006
1007 79259 static synode_no get_proposer_synode(int i) {
1008
3/6
✓ Branch 0 taken 79259 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 79259 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 79259 times.
✗ Branch 5 not taken.
 79259 if (i >= 0 && i < PROPOSERS && proposer_synodes[i]) {
1009 79259 return *proposer_synodes[i];
1010 } else {
1011 ✗ return null_synode;
1012 }
1013 }
1014
1015 6724 static synode_no min_proposer_synode() {
1016 synode_no s_min;
1017 int i;
1018
1/2
✓ Branch 0 taken 6724 times.
✗ Branch 1 not taken.
 6724 for (i = 0; i < PROPOSERS; i++) {
1019 6724 s_min = get_proposer_synode(i);
1020
1/2
✓ Branch 0 taken 6724 times.
✗ Branch 1 not taken.
 6724 if (!synode_eq(null_synode, s_min)) break; // Initial value
1021 }
1022
2/2
✓ Branch 0 taken 67240 times.
✓ Branch 1 taken 6724 times.
 73964 for (; i < PROPOSERS; i++) {
1023
2/2
✓ Branch 0 taken 5295 times.
✓ Branch 1 taken 61945 times.
 67240 if (synode_lt(get_proposer_synode(i), s_min))
1024 5295 s_min = get_proposer_synode(i);
1025 }
1026 6724 return s_min;
1027 }
1028
1029 2208 static void terminate_proposers() {
1030
2/2
✓ Branch 0 taken 22080 times.
✓ Branch 1 taken 2208 times.
 24288 PROP_ITER { task_terminate(proposer[i]); }
1031 2208 }
1032
1033 4643 static void free_forced_config_site_def() {
1034 4643 free_site_def(forced_config);
1035 4643 forced_config = nullptr;
1036 4643 }
1037
1038 #if TASK_DBUG_ON
1039 [[maybe_unused]] static void dbg_proposers();
1040 static void dbg_proposers() {
1041 GET_GOUT;
1042 if (!IS_XCOM_DEBUG_WITH(XCOM_DEBUG_TRACE)) return;
1043 NDBG(PROPOSERS, d);
1044 {
1045 PROP_ITER { PPUT(proposer[i]); }
1046 }
1047 PRINT_GOUT;
1048 FREE_GOUT;
1049 }
1050 #endif
1051
1052 2215 static void set_proposer_startpoint() {
1053 IFDBG(D_NONE, FN; STRLIT("changing current message"));
1054
2/2
✓ Branch 0 taken 1367 times.
✓ Branch 1 taken 848 times.
 2215 if (synode_gt(max_synode, get_current_message())) {
1055
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1367 times.
 1367 if (max_synode.msgno <= 1)
1056 ✗ set_current_message(first_free_synode_local(max_synode));
1057 else
1058 1367 set_current_message(incr_msgno(first_free_synode_local(max_synode)));
1059 }
1060
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2215 times.
 2215 if (synode_gt(executed_msg, get_current_message())) {
1061 ✗ set_current_message(first_free_synode_local(executed_msg));
1062 }
1063 2215 }
1064
1065 /* Task functions */
1066
1067 static xcom_state_change_cb xcom_run_cb = nullptr;
1068 static xcom_state_change_cb xcom_terminate_cb = nullptr;
1069 static xcom_state_change_cb xcom_comms_cb = nullptr;
1070 static xcom_state_change_cb xcom_exit_cb = nullptr;
1071 static xcom_state_change_cb xcom_expel_cb = nullptr;
1072 static xcom_input_try_pop_cb xcom_try_pop_from_input_cb = nullptr;
1073 static xcom_recovery_cb MY_ATTRIBUTE((unused)) recovery_begin_cb = nullptr;
1074 static xcom_recovery_cb MY_ATTRIBUTE((unused)) recovery_restart_cb = nullptr;
1075 static xcom_recovery_cb MY_ATTRIBUTE((unused)) recovery_init_cb = nullptr;
1076 static xcom_recovery_cb MY_ATTRIBUTE((unused)) recovery_end_cb = nullptr;
1077
1078 2292 void set_xcom_run_cb(xcom_state_change_cb x) { xcom_run_cb = x; }
1079 2292 void set_xcom_exit_cb(xcom_state_change_cb x) { xcom_exit_cb = x; }
1080 2292 void set_xcom_comms_cb(xcom_state_change_cb x) { xcom_comms_cb = x; }
1081 2292 void set_xcom_expel_cb(xcom_state_change_cb x) { xcom_expel_cb = x; }
1082
1083 2292 void set_xcom_input_try_pop_cb(xcom_input_try_pop_cb pop) {
1084 2292 xcom_try_pop_from_input_cb = pop;
1085 2292 }
1086
1087 #ifdef XCOM_STANDALONE
1088 /* purecov: begin deadcode */
1089 void set_xcom_terminate_cb(xcom_state_change_cb x) { xcom_terminate_cb = x; }
1090 /* purecov: end */
1091
1092 /* purecov: begin deadcode */
1093 void set_xcom_recovery_begin_cb(xcom_recovery_cb x) { recovery_begin_cb = x; }
1094 /* purecov: end */
1095
1096 /* purecov: begin deadcode */
1097 void set_xcom_recovery_restart_cb(xcom_recovery_cb x) {
1098 recovery_restart_cb = x;
1099 }
1100 /* purecov: end */
1101
1102 /* purecov: begin deadcode */
1103 void set_xcom_recovery_init_cb(xcom_recovery_cb x) { recovery_init_cb = x; }
1104 /* purecov: end */
1105
1106 /* purecov: begin deadcode */
1107 void set_xcom_recovery_end_cb(xcom_recovery_cb x) { recovery_end_cb = x; }
1108 /* purecov: end */
1109 #endif
1110
1111 /**
1112 * These fields are used to signal XCom's request queue. After a request
1113 * is added, one will write 1 byte to warn local_server_task that it has work to
1114 * do.
1115 *
1116 * We use two types of signalling connection:
1117 * - An anonymous pipe, when possible, in POSIX compatible systems
1118 * - A regular socket connection, in Windows
1119 *
1120 * input_signal_connection is the connection_descriptor returned when one opens
1121 * a local signalling connection. It will contain either:
1122 * - The write side of a connection, in case of using a pipe OR;
1123 * - A bidirectional connection, when using a regular socket connection;
1124 *
1125 * input_signal_connection_pipe is the connection_descriptor that holds the read
1126 * side of a pipe connection. It is only allocated when we are able to have
1127 * a pipe connection.
1128 */
1129
1130 static connection_descriptor *input_signal_connection{nullptr};
1131
1132 connection_descriptor *input_signal_connection_pipe{nullptr};
1133 int pipe_signal_connections[2] = {-1, -1};
1134
1135 #ifndef XCOM_WITHOUT_OPENSSL
1136 ✗ static bool_t xcom_input_signal_connection_shutdown_ssl_wait_for_peer() {
1137 ✗ int ssl_error_code = 0;
1138 do {
1139 char buf[1024];
1140 ✗ ssl_error_code = SSL_read(input_signal_connection->ssl_fd, buf, 1024);
1141 ✗ } while (ssl_error_code > 0);
1142
1143 {
1144 bool_t const successful =
1145 ✗ (SSL_get_error(input_signal_connection->ssl_fd, ssl_error_code) ==
1146 ✗ SSL_ERROR_ZERO_RETURN);
1147 ✗ return successful;
1148 }
1149 }
1150
1151 ✗ static bool_t xcom_input_signal_connection_shutdown_ssl() {
1152 ✗ bool_t successful = FALSE;
1153
1154 ✗ int ssl_error_code = SSL_shutdown(input_signal_connection->ssl_fd);
1155
1156 ✗ bool_t const need_to_wait_for_peer_shutdown = (ssl_error_code == 0);
1157 ✗ bool_t const something_went_wrong = (ssl_error_code < 0);
1158 ✗ if (need_to_wait_for_peer_shutdown) {
1159 ✗ successful = xcom_input_signal_connection_shutdown_ssl_wait_for_peer();
1160 ✗ if (!successful) goto end;
1161 ✗ } else if (something_went_wrong) {
1162 ✗ goto end;
1163 }
1164
1165 ✗ ssl_free_con(input_signal_connection);
1166 ✗ successful = TRUE;
1167
1168 ✗ end:
1169 ✗ return successful;
1170 }
1171 #endif
1172
1173 2385 bool_t xcom_input_new_signal_connection(char const *address, xcom_port port) {
1174 2385 bool_t const SUCCESSFUL = TRUE;
1175 2385 bool_t const UNSUCCESSFUL = FALSE;
1176
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2385 times.
 2385 assert(input_signal_connection == nullptr);
1177
1178
1/2
✓ Branch 0 taken 2385 times.
✗ Branch 1 not taken.
 2385 if (input_signal_connection_pipe != nullptr) {
1179 2385 input_signal_connection =
1180 2385 (connection_descriptor *)malloc(sizeof(connection_descriptor));
1181 2385 input_signal_connection->fd = pipe_signal_connections[1];
1182 #ifndef XCOM_WITHOUT_OPENSSL
1183 2385 input_signal_connection->ssl_fd = nullptr;
1184 #endif
1185 2385 set_connected(input_signal_connection, CON_FD);
1186
1187
2/4
✓ Branch 0 taken 2385 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2385 times.
✗ Branch 3 not taken.
 2385 G_INFO("Successfully connected to the local XCom via anonymous pipe");
1188
1189 2385 return SUCCESSFUL;
1190 } else {
1191 /* purecov: begin deadcode */
1192 /* Try to connect. */
1193 ✗ input_signal_connection = open_new_local_connection(address, port);
1194 ✗ if (input_signal_connection->fd == -1) {
1195 ✗ return UNSUCCESSFUL;
1196 }
1197
1198 /* Have the server handle the rest of this connection using a local_server
1199 task. */
1200 ✗ if (xcom_client_convert_into_local_server(input_signal_connection) == 1) {
1201 ✗ G_TRACE(
1202 "Converted the signalling connection handler into a local_server "
1203 "task on the client side.");
1204
1205 #ifndef XCOM_WITHOUT_OPENSSL
1206 /* No more SSL in this connection. */
1207 ✗ if (Network_provider_manager::getInstance().get_running_protocol() ==
1208 XCOM_PROTOCOL) {
1209 ✗ bool_t const using_ssl = (input_signal_connection->ssl_fd != nullptr);
1210 ✗ if (using_ssl) {
1211 ✗ bool_t successful = xcom_input_signal_connection_shutdown_ssl();
1212 ✗ if (!successful) {
1213 ✗ G_ERROR(
1214 "Error shutting down SSL on XCom's signalling connection on "
1215 "the "
1216 "client side.");
1217 ✗ xcom_input_free_signal_connection();
1218 ✗ return UNSUCCESSFUL;
1219 }
1220 }
1221 }
1222 #endif
1223 ✗ G_INFO("Successfully connected to the local XCom via socket connection");
1224 ✗ return SUCCESSFUL;
1225 } else {
1226 ✗ G_INFO(
1227 "Error converting the signalling connection handler into a "
1228 "local_server task on the client side. This will result on a failure "
1229 "to join this node to a configuration");
1230 ✗ xcom_input_free_signal_connection();
1231 ✗ return UNSUCCESSFUL;
1232 }
1233 /* purecov: end */
1234 }
1235 }
1236
1237 79614 bool_t xcom_input_signal() {
1238 79614 bool_t successful = FALSE;
1239
2/2
✓ Branch 0 taken 79594 times.
✓ Branch 1 taken 20 times.
 79614 if (input_signal_connection != nullptr) {
1240 79594 unsigned char tiny_buf[1] = {0};
1241 int64_t error_code;
1242 79594 connnection_write_method to_write_function =
1243
1/2
✓ Branch 0 taken 79594 times.
✗ Branch 1 not taken.
 79594 input_signal_connection_pipe != nullptr ? con_pipe_write : con_write;
1244
1245 error_code =
1246
1/2
✓ Branch 0 taken 79594 times.
✗ Branch 1 not taken.
 79594 socket_write(input_signal_connection, tiny_buf, 1, to_write_function);
1247
1248 79594 successful = (error_code == 1);
1249 }
1250 79614 return successful;
1251 }
1252
1253 7050 void xcom_input_free_signal_connection() {
1254
2/2
✓ Branch 0 taken 2378 times.
✓ Branch 1 taken 4672 times.
 7050 if (input_signal_connection != nullptr) {
1255
1/2
✓ Branch 0 taken 2378 times.
✗ Branch 1 not taken.
 2378 if (input_signal_connection_pipe != nullptr) {
1256 2378 close(input_signal_connection->fd);
1257 } else {
1258 /* purecov: begin deadcode */
1259 ✗ close_open_connection(input_signal_connection);
1260 /* purecov: end */
1261 }
1262
1263 2378 free(input_signal_connection);
1264 2378 input_signal_connection = nullptr;
1265 }
1266 7050 }
1267
1268 #ifndef XCOM_WITHOUT_OPENSSL
1269 ✗ static int local_server_shutdown_ssl(connection_descriptor *con, void *buf,
1270 int n, int *ret) {
1271 DECL_ENV
1272 int ssl_error_code;
1273 bool_t need_to_wait_for_peer_shutdown;
1274 bool_t something_went_wrong;
1275 int64_t nr_read;
1276 ✗ ENV_INIT
1277 ✗ END_ENV_INIT
1278 END_ENV;
1279 ✗ *ret = 0;
1280 ✗ TASK_BEGIN
1281 ✗ ep->ssl_error_code = SSL_shutdown(con->ssl_fd);
1282 ✗ ep->need_to_wait_for_peer_shutdown = (ep->ssl_error_code == 0);
1283 ✗ ep->something_went_wrong = (ep->ssl_error_code < 0);
1284 ✗ if (ep->need_to_wait_for_peer_shutdown) {
1285 do {
1286 ✗ TASK_CALL(task_read(con, buf, n, &ep->nr_read));
1287 ✗ } while (ep->nr_read > 0);
1288 ✗ ep->ssl_error_code =
1289 ✗ SSL_get_error(con->ssl_fd, static_cast<int>(ep->nr_read));
1290 ✗ ep->something_went_wrong = (ep->ssl_error_code != SSL_ERROR_ZERO_RETURN);
1291 }
1292 ✗ if (ep->something_went_wrong) TERMINATE;
1293 ✗ ssl_free_con(con);
1294 ✗ *ret = 1;
1295 ✗ FINALLY
1296 ✗ TASK_END;
1297 }
1298 #endif
1299
1300 83804 int local_server(task_arg arg) {
1301 DECL_ENV
1302 connection_descriptor rfd;
1303 int ssl_shutdown_ret;
1304 unsigned char buf[1024]; /* arbitrary size */
1305 int64_t nr_read;
1306 xcom_input_request_ptr request;
1307 xcom_input_request_ptr next_request;
1308 pax_msg *request_pax_msg;
1309 pax_msg *reply_payload;
1310 linkage internal_reply_queue;
1311 msg_link *internal_reply;
1312 bool signaling_connection_error;
1313 connnection_read_method signal_read;
1314 2385 ENV_INIT
1315 2385 rfd.fd = -1;
1316 2385 ssl_shutdown_ret = 0;
1317 2385 memset(buf, 0, 1024);
1318 2385 nr_read = 0;
1319 2385 request = nullptr;
1320 2385 link_init(&internal_reply_queue, TYPE_HASH("msg_link"));
1321 2385 next_request = nullptr;
1322 2385 request_pax_msg = nullptr;
1323 2385 reply_payload = nullptr;
1324 2385 internal_reply = nullptr;
1325 2385 signaling_connection_error = false;
1326 2385 END_ENV_INIT
1327 END_ENV;
1328
6/11
✓ Branch 0 taken 2385 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 81419 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 2385 times.
✓ Branch 7 taken 2385 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 8 times.
✓ Branch 10 taken 2377 times.
 83804 TASK_BEGIN
1329
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2377 times.
 2377 assert(xcom_try_pop_from_input_cb != nullptr);
1330 {
1331 2377 connection_descriptor *arg_rfd = (connection_descriptor *)get_void_arg(arg);
1332 2377 ep->rfd = *arg_rfd;
1333
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2377 times.
 2377 if (input_signal_connection_pipe == nullptr) free(arg_rfd);
1334 }
1335
1336 // We will check if we have a pipe open or if we use a classic signalling
1337 // connection.
1338 2377 ep->signal_read =
1339
1/2
✓ Branch 0 taken 2377 times.
✗ Branch 1 not taken.
 2377 input_signal_connection_pipe != nullptr ? con_pipe_read : con_read;
1340
1341 #ifndef XCOM_WITHOUT_OPENSSL
1342 /* No more SSL in this connection. */
1343 2377 if (Network_provider_manager::getInstance().get_running_protocol() ==
1344
3/4
✓ Branch 0 taken 2310 times.
✓ Branch 1 taken 67 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2377 times.
 4687 XCOM_PROTOCOL &&
1345
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2310 times.
 2310 ep->rfd.ssl_fd) {
1346 ✗ TASK_CALL(local_server_shutdown_ssl(&ep->rfd, ep->buf, 1024,
1347 &ep->ssl_shutdown_ret));
1348 ✗ if (ep->ssl_shutdown_ret != 1) {
1349 ✗ G_ERROR(
1350 "Error shutting down SSL on XCom's signalling connection on the "
1351 "server side.");
1352 ✗ TERMINATE;
1353 }
1354 }
1355 #endif
1356
1357
1/2
✓ Branch 0 taken 81599 times.
✗ Branch 1 not taken.
 81599 while (!xcom_shutdown) {
1358 /* Wait for signal that there is work to consume from the queue. */
1359
1/2
✓ Branch 0 taken 81599 times.
✗ Branch 1 not taken.
 81599 if (!ep->signaling_connection_error) {
1360
10/14
✓ Branch 0 taken 81592 times.
✓ Branch 1 taken 81426 times.
✓ Branch 2 taken 2370 times.
✓ Branch 3 taken 79222 times.
✓ Branch 4 taken 81426 times.
✓ Branch 5 taken 79222 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 81419 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 81419 times.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✓ Branch 12 taken 81419 times.
✓ Branch 13 taken 79222 times.
 242240 TASK_CALL(
1361 task_read(&ep->rfd, ep->buf, 1024, &ep->nr_read, ep->signal_read));
1362
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 79222 times.
 79222 if (ep->nr_read == 0) {
1363 /* purecov: begin inspected */
1364 ✗ G_WARNING("local_server: client closed the signalling connection?");
1365 ✗ ep->signaling_connection_error = true;
1366 /* purecov: end */
1367
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 79222 times.
 79222 } else if (ep->nr_read < 0) {
1368 /* purecov: begin inspected */
1369 IFDBG(D_NONE, FN; NDBG64(ep->nr_read));
1370 ✗ G_WARNING(
1371 "local_server: error reading from the signalling connection?");
1372 ✗ ep->signaling_connection_error = true;
1373 /* purecov: end */
1374 }
1375 }
1376
1377 /**
1378 * If an error occurs or if the client connection for the local server is
1379 * forcefully shutdown, we continue processing the queue until the end
1380 * resorting to time-based waits.
1381 */
1382
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 79222 times.
 79222 if (ep->signaling_connection_error) {
1383 ✗ TASK_DELAY(0.1);
1384 }
1385
1386 /* Pop, dispatch, and reply. */
1387 79222 ep->request = xcom_try_pop_from_input_cb();
1388
2/2
✓ Branch 0 taken 79328 times.
✓ Branch 1 taken 79222 times.
 158550 while (ep->request != nullptr) {
1389 /* Take ownership of the tail of the list, otherwise we lose it when we
1390 free ep->request. */
1391 79328 ep->next_request = xcom_input_request_extract_next(ep->request);
1392 79328 unchecked_replace_pax_msg(&ep->request_pax_msg,
1393 pax_msg_new_0(null_synode));
1394
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 79328 times.
 79328 assert(ep->request_pax_msg->refcnt == 1);
1395 79328 ep->request_pax_msg->op = client_msg;
1396
1397 /* Take ownership of the request's app_data, otherwise the app_data is
1398 freed with ep->request. */
1399 79328 ep->request_pax_msg->a = xcom_input_request_extract_app_data(ep->request);
1400 79328 ep->request_pax_msg->to = VOID_NODE_NO;
1401 79328 ep->request_pax_msg->force_delivery =
1402 79328 (ep->request_pax_msg->a->body.c_t == force_config_type);
1403 79328 dispatch_op(nullptr, ep->request_pax_msg, &ep->internal_reply_queue);
1404
2/2
✓ Branch 0 taken 2546 times.
✓ Branch 1 taken 76782 times.
 79328 if (!link_empty(&ep->internal_reply_queue)) {
1405 2546 ep->internal_reply =
1406 2546 (msg_link *)(link_extract_first(&ep->internal_reply_queue));
1407
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2546 times.
 2546 assert(ep->internal_reply->p);
1408
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2546 times.
 2546 assert(ep->internal_reply->p->refcnt == 1);
1409 /* We are going to take ownership of the pax_msg which has the reply
1410 payload, so we bump its reference count so that it is not freed by
1411 msg_link_delete. */
1412 2546 ep->reply_payload = ep->internal_reply->p;
1413 2546 ep->reply_payload->refcnt++;
1414 2546 msg_link_delete(&ep->internal_reply);
1415 /* There should only have been one reply. */
1416
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2546 times.
 2546 assert(link_empty(&ep->internal_reply_queue));
1417 } else {
1418 76782 ep->reply_payload = nullptr;
1419 }
1420 /* Reply to the request. */
1421 79328 xcom_input_request_reply(ep->request, ep->reply_payload);
1422 79328 xcom_input_request_free(ep->request);
1423 79328 ep->request = ep->next_request;
1424 }
1425 }
1426 ✗ FINALLY
1427 IFDBG(D_BUG, FN; STRLIT(" shutdown "); NDBG(ep->rfd.fd, d);
1428 NDBG(task_now(), f));
1429 /* Close the signalling connection. */
1430
1/2
✓ Branch 0 taken 2378 times.
✗ Branch 1 not taken.
 2378 if (!ep->signaling_connection_error) {
1431
1/2
✓ Branch 0 taken 2378 times.
✗ Branch 1 not taken.
 2378 if (input_signal_connection_pipe != nullptr &&
1432
2/2
✓ Branch 0 taken 2370 times.
✓ Branch 1 taken 8 times.
 2378 ep->rfd.fd != -1) { // We add -1 here, because in rare cases, the task
1433 // might have not been activated. Thus, it might
1434 // not have a reference to the socket to close.
1435 2370 close(ep->rfd.fd);
1436 2370 remove_and_wakeup(ep->rfd.fd);
1437 } else {
1438 8 shutdown_connection(&ep->rfd);
1439 }
1440 }
1441
1442 2378 unchecked_replace_pax_msg(&ep->request_pax_msg, nullptr);
1443 IFDBG(D_NONE, FN; NDBG(xcom_shutdown, d));
1444
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2378 times.
 2378 TASK_END;
1445 }
1446
1447 2385 static bool_t local_server_is_setup() {
1448 2385 return xcom_try_pop_from_input_cb != nullptr;
1449 }
1450
1451 static void init_time_queue();
1452 static int paxos_timer_task(task_arg arg [[maybe_unused]]);
1453
1454 2385 int xcom_taskmain2(xcom_port listen_port) {
1455 2385 init_xcom_transport(listen_port);
1456
1457 IFDBG(D_BUG, FN; STRLIT("enter taskmain"));
1458 2385 ignoresig(SIGPIPE);
1459
1460 {
1461 2385 result tcp_fd = {0, 0};
1462
1463 /*
1464 Setup networking
1465 */
1466
1/2
✓ Branch 0 taken 2385 times.
✗ Branch 1 not taken.
 2385 auto &net_manager = Network_provider_manager::getInstance();
1467 bool error_starting_network_provider =
1468
1/2
✓ Branch 0 taken 2385 times.
✗ Branch 1 not taken.
 2385 net_manager.start_active_network_provider();
1469
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2385 times.
 2385 if (error_starting_network_provider) {
1470 /* purecov: begin inspected */
1471 ✗ g_critical("Unable to start %s Network Provider",
1472 Communication_stack_to_string::to_string(
1473 net_manager.get_running_protocol()));
1474 ✗ if (xcom_comms_cb) {
1475 ✗ xcom_comms_cb(XCOM_COMMS_ERROR);
1476 }
1477 ✗ if (xcom_terminate_cb) {
1478 ✗ xcom_terminate_cb(0);
1479 }
1480 ✗ goto cleanup;
1481 /* purecov: end */
1482 }
1483
1484 // We will use POSIX pipes for local queue signaling if we are not using WIN32
1485 #if !defined(_WIN32)
1486
1/2
✓ Branch 0 taken 2385 times.
✗ Branch 1 not taken.
 2385 if (local_server_is_setup()) {
1487 /* Launch local_server task to handle this connection. */
1488 {
1489
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2385 times.
 2385 if (pipe(pipe_signal_connections) == -1) {
1490 /* purecov: begin inspected */
1491 ✗ g_critical("Unable to start local signaling mechanism");
1492 ✗ if (xcom_comms_cb) {
1493 ✗ xcom_comms_cb(XCOM_COMMS_ERROR);
1494 }
1495 ✗ if (xcom_terminate_cb) {
1496 ✗ xcom_terminate_cb(0);
1497 }
1498 ✗ goto cleanup;
1499 /* purecov: end */
1500 }
1501
1/2
✓ Branch 0 taken 2385 times.
✗ Branch 1 not taken.
 2385 unblock_fd(pipe_signal_connections[0]);
1502
1503 /*
1504 Create the read side of input_signal_connection_pipe and create the
1505 local_server.
1506
1507 If one would to use regular sockets, this code is not executed and
1508 the local_server is created in the dispatch_op function.
1509 */
1510 2385 input_signal_connection_pipe =
1511 2385 (connection_descriptor *)malloc(sizeof(connection_descriptor));
1512 2385 input_signal_connection_pipe->fd = pipe_signal_connections[0];
1513 #ifndef XCOM_WITHOUT_OPENSSL
1514 2385 input_signal_connection_pipe->ssl_fd = nullptr;
1515 #endif
1516 2385 set_connected(input_signal_connection_pipe, CON_FD);
1517
1/2
✓ Branch 0 taken 2385 times.
✗ Branch 1 not taken.
 2385 task_new(local_server, void_arg(input_signal_connection_pipe),
1518 "local_server", XCOM_THREAD_DEBUG);
1519 }
1520 }
1521 #endif
1522
1523
1/2
✓ Branch 0 taken 2385 times.
✗ Branch 1 not taken.
 2385 if (xcom_comms_cb) {
1524
1/2
✓ Branch 0 taken 2385 times.
✗ Branch 1 not taken.
 2385 xcom_comms_cb(XCOM_COMMS_OK);
1525 }
1526
1527 IFDBG(D_NONE, FN; STRLIT("Creating tasks"));
1528
1529
1/2
✓ Branch 0 taken 2385 times.
✗ Branch 1 not taken.
 2385 task_new(incoming_connection_task, int_arg(tcp_fd.val), "tcp_server",
1530 XCOM_THREAD_DEBUG);
1531
1/2
✓ Branch 0 taken 2385 times.
✗ Branch 1 not taken.
 2385 task_new(tcp_reaper_task, null_arg, "tcp_reaper_task", XCOM_THREAD_DEBUG);
1532 #if defined(_WIN32)
1533 task_new(tcp_reconnection_task, null_arg, "tcp_reconnection_task",
1534 XCOM_THREAD_DEBUG);
1535 #endif
1536
1537 2385 init_time_queue();
1538
1/2
✓ Branch 0 taken 2385 times.
✗ Branch 1 not taken.
 2385 task_new(paxos_timer_task, null_arg, "paxos_timer_task", XCOM_THREAD_DEBUG);
1539 IFDBG(D_BUG, FN; STRLIT("XCOM is listening on "); NPUT(listen_port, d));
1540 }
1541
1542 #ifdef XCOM_STANDALONE
1543 if (recovery_init_cb) recovery_init_cb();
1544
1545 if (recovery_begin_cb) recovery_begin_cb();
1546 #endif
1547
1548 2385 task_loop();
1549 2378 cleanup:
1550
1551 #ifdef TASK_EVENT_TRACE
1552 dump_task_events();
1553 #endif
1554 // STOP NETWORK PROVIDERS
1555 2378 Network_provider_manager::getInstance().stop_all_network_providers();
1556
1557 2378 xcom_thread_deinit();
1558
1559 IFDBG(D_BUG, FN; STRLIT(" exit "); NDBG(xcom_dbg_stack_top, d);
1560 NDBG((unsigned)xcom_debug_mask, x));
1561 2378 xcom_debug_mask = 0;
1562 2378 xcom_dbg_stack_top = 0;
1563
1/2
✓ Branch 0 taken 2378 times.
✗ Branch 1 not taken.
 2378 if (input_signal_connection_pipe != nullptr) {
1564 2378 ::xcom_input_free_signal_connection();
1565
1566 2378 free(input_signal_connection_pipe);
1567 2378 input_signal_connection_pipe = nullptr;
1568
1569 2378 pipe_signal_connections[0] = -1;
1570 2378 pipe_signal_connections[1] = -1;
1571 }
1572
1573
1/2
✓ Branch 0 taken 2378 times.
✗ Branch 1 not taken.
 2378 if (xcom_exit_cb) {
1574 2378 xcom_exit_cb(0);
1575 }
1576
1577 2378 return 1;
1578 }
1579
1580 /* Paxos message construction and sending */
1581
1582 /* Initialize a message for sending */
1583 634940 static void prepare(pax_msg *p, pax_op op) {
1584 634940 p->op = op;
1585 634940 p->reply_to = p->proposal;
1586 634940 }
1587
1588 /* Initialize a prepare_msg */
1589 42238 void init_prepare_msg(pax_msg *p) { prepare(p, prepare_op); }
1590
1591 21917 static int prepare_msg(pax_msg *p) {
1592 21917 init_prepare_msg(p);
1593 /* p->msg_type = normal; */
1594 21917 return send_to_acceptors(p, "prepare_msg");
1595 }
1596
1597 /* Initialize a noop_msg */
1598 20320 pax_msg *create_noop(pax_msg *p) {
1599 20320 init_prepare_msg(p);
1600 20320 p->msg_type = no_op;
1601 20320 return p;
1602 }
1603
1604 /* Initialize a read_msg */
1605 436131 static pax_msg *create_read(site_def const *site, pax_msg *p) {
1606 436131 p->msg_type = normal;
1607 436131 p->proposal.node = get_nodeno(site);
1608 436131 prepare(p, read_op);
1609 436131 return p;
1610 }
1611
1612 156571 static int skip_msg(pax_msg *p) {
1613 156571 prepare(p, skip_op);
1614 IFDBG(D_NONE, FN; STRLIT("skipping message "); SYCEXP(p->synode));
1615 156571 p->msg_type = no_op;
1616 156571 return send_to_all(p, "skip_msg");
1617 }
1618
1619 103304 static void brand_app_data(pax_msg *p) {
1620 103304 app_data_ptr a = p->a;
1621
2/2
✓ Branch 0 taken 84896 times.
✓ Branch 1 taken 103304 times.
 188200 while (a) {
1622 84896 a->app_key = p->synode;
1623 84896 a->group_id = p->synode.group_id;
1624 IFDBG(D_NONE, FN; PTREXP(a); SYCEXP(p->synode); SYCEXP(a->app_key));
1625 84896 a = a->next;
1626 }
1627 103304 }
1628
1629 83694 static synode_no my_unique_id(synode_no synode) {
1630
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 83694 times.
 83694 assert(my_id != 0);
1631 83694 site_def const *site = find_site_def(synode);
1632 /* Random number derived from node number and timestamp which uniquely defines
1633 * this instance */
1634 83694 synode.group_id = my_id;
1635 83694 synode.node = get_nodeno(site);
1636 83694 return synode;
1637 }
1638
1639 83694 static void set_unique_id(pax_msg *msg, synode_no synode) {
1640 83694 app_data_ptr a = msg->a;
1641
2/2
✓ Branch 0 taken 84204 times.
✓ Branch 1 taken 83694 times.
 167898 while (a) {
1642 84204 a->unique_id = synode;
1643 84204 a = a->next;
1644 }
1645 83694 }
1646
1647 103304 void init_propose_msg(pax_msg *p) {
1648 103304 p->op = accept_op;
1649 103304 p->reply_to = p->proposal;
1650 103304 brand_app_data(p);
1651 /* set_unique_id(p, my_unique_id(synode)); */
1652 103304 }
1653
1654 93579 static int send_propose_msg(pax_msg *p) {
1655 93579 return send_to_acceptors(p, "propose_msg");
1656 }
1657
1658 83638 static int propose_msg(pax_msg *p) {
1659 83638 init_propose_msg(p);
1660 83638 return send_propose_msg(p);
1661 }
1662
1663 121900 static void set_learn_type(pax_msg *p) {
1664 121900 p->op = learn_op;
1665 121900 p->msg_type = p->a ? normal : no_op;
1666 121900 }
1667
1668 /* purecov: begin deadcode */
1669 ✗ static void init_learn_msg(pax_msg *p) {
1670 ✗ set_learn_type(p);
1671 ✗ p->reply_to = p->proposal;
1672 }
1673
1674 ✗ static int send_learn_msg(site_def const *site, pax_msg *p) {
1675 IFDBG(D_NONE, FN; dbg_bitset(p->receivers, get_maxnodes(site)););
1676 ✗ return send_to_all_site(site, p, "learn_msg");
1677 }
1678 /* purecov: end */
1679
1680 91543 static pax_msg *create_tiny_learn_msg(pax_machine *pm, pax_msg *p) {
1681 91543 pax_msg *tiny_learn_msg = clone_pax_msg_no_app(p);
1682
1683 91543 ref_msg(tiny_learn_msg);
1684 91543 tiny_learn_msg->msg_type = p->a ? normal : no_op;
1685 91543 tiny_learn_msg->op = tiny_learn_op;
1686 91543 tiny_learn_msg->reply_to = pm->proposer.bal;
1687
1688 91543 return tiny_learn_msg;
1689 }
1690
1691 91541 static int send_tiny_learn_msg(site_def const *site, pax_msg *p) {
1692 91541 int retval = send_to_all_site(site, p, "tiny_learn_msg");
1693 91541 unref_msg(&p);
1694 91541 return retval;
1695 }
1696
1697 /* Proposer task */
1698
1699 21918 void prepare_push_3p(site_def const *site, pax_machine *p, pax_msg *msg,
1700 synode_no msgno, pax_msg_type msg_type) {
1701 IFDBG(D_NONE, FN; SYCEXP(msgno); NDBG(p->proposer.bal.cnt, d);
1702 NDBG(p->acceptor.promise.cnt, d));
1703 21918 BIT_ZERO(p->proposer.prep_nodeset);
1704 21918 p->proposer.bal.node = get_nodeno(site);
1705 {
1706
2/2
✓ Branch 0 taken 1335 times.
✓ Branch 1 taken 20583 times.
 21918 int maxcnt = MAX(p->proposer.bal.cnt, p->acceptor.promise.cnt);
1707 21918 p->proposer.bal.cnt = ++maxcnt;
1708 }
1709 21918 msg->synode = msgno;
1710 21918 msg->proposal = p->proposer.bal;
1711 21918 msg->msg_type = msg_type;
1712 21918 msg->force_delivery = p->force_delivery;
1713 21918 }
1714
1715 83639 void prepare_push_2p(site_def const *site, pax_machine *p) {
1716
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 83639 times.
 83639 assert(p->proposer.msg);
1717
1718 83639 BIT_ZERO(p->proposer.prop_nodeset);
1719 IFDBG(D_NONE, FN; SYCEXP(p->synode));
1720 83639 p->proposer.bal.cnt = 0;
1721 83639 p->proposer.bal.node = get_nodeno(site);
1722 83639 p->proposer.msg->proposal = p->proposer.bal;
1723 83639 p->proposer.msg->synode = p->synode;
1724 83639 p->proposer.msg->force_delivery = p->force_delivery;
1725 83639 }
1726
1727 83638 static void push_msg_2p(site_def const *site, pax_machine *p) {
1728 83638 prepare_push_2p(site, p);
1729 83638 propose_msg(p->proposer.msg);
1730 83638 }
1731
1732 21917 static void push_msg_3p(site_def const *site, pax_machine *p, pax_msg *msg,
1733 synode_no msgno, pax_msg_type msg_type) {
1734
2/2
✓ Branch 0 taken 392 times.
✓ Branch 1 taken 21525 times.
 21917 if (wait_forced_config) {
1735 392 force_pax_machine(p, 1);
1736 }
1737
1738
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 21917 times.
 21917 assert(msgno.msgno != 0);
1739 21917 prepare_push_3p(site, p, msg, msgno, msg_type);
1740
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 21917 times.
 21917 assert(p->proposer.msg);
1741 21917 prepare_msg(msg);
1742 IFDBG(D_NONE, FN; BALCEXP(msg->proposal); SYCEXP(msgno); STRLIT(" op ");
1743 STRLIT(pax_op_to_str(msg->op)));
1744 21917 }
1745
1746 /* Brand client message with unique ID */
1747 83694 static void brand_client_msg(pax_msg *msg, synode_no msgno) {
1748
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 83694 times.
 83694 assert(!synode_eq(msgno, null_synode));
1749 83694 set_unique_id(msg, my_unique_id(msgno));
1750 83694 }
1751
1752 6364 void xcom_send(app_data_ptr a, pax_msg *msg) {
1753 IFDBG(D_NONE, FN; PTREXP(a); SYCEXP(a->app_key); SYCEXP(msg->synode));
1754 6364 msg->a = a;
1755 6364 msg->op = client_msg;
1756 {
1757 6364 msg_link *link = msg_link_new(msg, VOID_NODE_NO);
1758 IFDBG(D_NONE, FN; COPY_AND_FREE_GOUT(dbg_pax_msg(msg)));
1759 6364 channel_put(&prop_input_queue, &link->l);
1760 }
1761 6364 }
1762
1763 #define FNVSTART 0x811c9dc5
1764
1765 /* Fowler-Noll-Vo type multiplicative hash */
1766 10269 static uint32_t fnv_hash(unsigned char *buf, size_t length, uint32_t sum) {
1767 10269 size_t i = 0;
1768
2/2
✓ Branch 0 taken 1389738 times.
✓ Branch 1 taken 10269 times.
 1400007 for (i = 0; i < length; i++) {
1769 1389738 sum = sum * (uint32_t)0x01000193 ^ (uint32_t)buf[i];
1770 }
1771 10269 return sum;
1772 }
1773
1774 /**
1775 Create a new (hopefully unique) ID. The basic idea is to create a hash from
1776 the host ID and a timestamp.
1777 */
1778 3423 uint32_t new_id() {
1779
1/2
✓ Branch 0 taken 3423 times.
✗ Branch 1 not taken.
 3423 long id = xcom_unique_long();
1780
1/2
✓ Branch 0 taken 3423 times.
✗ Branch 1 not taken.
 3423 double timestamp = task_now();
1781 3423 uint32_t retval = 0;
1782
5/6
✓ Branch 0 taken 3423 times.
✓ Branch 1 taken 3423 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 3423 times.
✓ Branch 4 taken 3423 times.
✓ Branch 5 taken 3423 times.
 10269 while (retval == 0 ||
1783 3423 is_dead_site(retval)) { /* Avoid returning 0 or already used site id */
1784 3423 retval = fnv_hash((unsigned char *)&id, sizeof(id), 0);
1785 3423 retval = fnv_hash((unsigned char *)&timestamp, sizeof(timestamp), retval);
1786 }
1787 3423 return retval;
1788 }
1789
1790 7766 static synode_no getstart(app_data_ptr a) {
1791 7766 synode_no retval = null_synode;
1792 /* If a->group_id is null_id, we set the group id from app_key.group_id,
1793 * which is hopefully not null_id. If it is, we're out of luck. */
1794
2/4
✓ Branch 0 taken 7766 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 7766 times.
 7766 if (a && a->group_id == null_id) {
1795 /* purecov: begin deadcode */
1796 ✗ a->group_id = a->app_key.group_id; /* app_key may have valid group */
1797 /* purecov: end */
1798 }
1799
2/2
✓ Branch 0 taken 13 times.
✓ Branch 1 taken 7753 times.
 7766 G_DEBUG("pid %d getstart group_id %x", xpid(), a->group_id);
1800
2/4
✓ Branch 0 taken 7766 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 7766 times.
 7766 if (!a || a->group_id == null_id) {
1801 ✗ retval.group_id = new_id();
1802 } else {
1803 7766 a->app_key.group_id = a->group_id;
1804 7766 retval = a->app_key;
1805
4/4
✓ Branch 0 taken 6918 times.
✓ Branch 1 taken 848 times.
✓ Branch 2 taken 6061 times.
✓ Branch 3 taken 1705 times.
 14684 if (get_site_def() &&
1806
2/2
✓ Branch 0 taken 6061 times.
✓ Branch 1 taken 857 times.
 6918 retval.msgno > 1) { /* Special case for initial boot of site */
1807 /* Not valid until after event horizon has been passed */
1808 6061 retval = add_event_horizon(retval);
1809 }
1810 }
1811 7766 return retval;
1812 }
1813
1814 #ifdef PERMISSIVE_EH_ACTIVE_CONFIG
1815 /* purecov: begin deadcode */
1816 synode_no get_default_start(app_data_ptr a) {
1817 synode_no retval = null_synode;
1818 /* If a->group_id is null_id, we set the group id from app_key.group_id,
1819 * which is hopefully not null_id. If it is, we're out of luck. */
1820 if (a && a->group_id == null_id) {
1821 a->group_id = a->app_key.group_id; /* app_key may have valid group */
1822 }
1823 G_DEBUG("pid %d getstart group_id %x", xpid(), a ? a->group_id : 0);
1824 if (!a || a->group_id == null_id) {
1825 retval.group_id = new_id();
1826 } else {
1827 a->app_key.group_id = a->group_id;
1828 retval = a->app_key;
1829 if (retval.msgno > 1) { /* Special case for initial boot of site */
1830 /* Not valid until after event horizon has been passed */
1831 retval = add_default_event_horizon(retval);
1832 }
1833 }
1834 return retval;
1835 }
1836 /* purecov: end */
1837 #endif
1838
1839 #if TASK_DBUG_ON
1840 /* purecov: begin deadcode */
1841 static void dump_xcom_node_names(site_def const *site) {
1842 u_int i;
1843 constexpr const size_t bufsize = NSERVERS * 256;
1844 char buf[bufsize]; /* Big enough */
1845 char *p = buf;
1846 if (!site) {
1847 G_INFO("pid %d no site", xpid());
1848 return;
1849 }
1850 *p = 0;
1851 for (i = 0; i < site->nodes.node_list_len; i++) {
1852 p = strncat(p, site->nodes.node_list_val[i].address, bufsize - 1);
1853 p = strncat(p, " ", bufsize - 1);
1854 }
1855 buf[bufsize - 1] = 0;
1856 G_INFO("pid %d node names %s", xpid(), buf);
1857 }
1858 /* purecov: end */
1859 #endif
1860
1861 12399 void site_install_action(site_def *site, cargo_type operation) {
1862 IFDBG(D_NONE, FN; NDBG(get_nodeno(get_site_def()), u));
1863
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 12399 times.
 12399 assert(site->event_horizon);
1864
4/4
✓ Branch 0 taken 11546 times.
✓ Branch 1 taken 853 times.
✓ Branch 2 taken 8049 times.
✓ Branch 3 taken 4350 times.
 23945 if (group_mismatch(site->start, max_synode) ||
1865
2/2
✓ Branch 0 taken 7196 times.
✓ Branch 1 taken 4350 times.
 11546 synode_gt(site->start, max_synode))
1866 8049 set_max_synode(site->start);
1867 12399 site->nodeno = xcom_find_node_index(&site->nodes);
1868 12399 push_site_def(site);
1869 IFDBG(D_NONE, dump_xcom_node_names(site));
1870 IFDBG(D_BUG, FN; SYCEXP(site->start); SYCEXP(site->boot_key);
1871 NUMEXP(site->max_active_leaders));
1872 IFDBG(D_BUG, FN; COPY_AND_FREE_GOUT(dbg_site_def(site)));
1873 12399 set_group(get_group_id(site));
1874
2/2
✓ Branch 0 taken 11511 times.
✓ Branch 1 taken 888 times.
 12399 if (get_maxnodes(get_site_def())) {
1875 11511 update_servers(site, operation);
1876 }
1877 12399 site->install_time = task_now();
1878
3/6
✓ Branch 0 taken 12399 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 12399 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 12399 times.
✗ Branch 5 not taken.
 12399 G_INFO(
1879 "Sucessfully installed new site definition. Start synode for this "
1880 "configuration is " SY_FMT ", boot key synode is " SY_FMT
1881 ", configured event horizon=%" PRIu32 ", my node identifier is %u",
1882 SY_MEM(site->start), SY_MEM(site->boot_key), site->event_horizon,
1883 get_nodeno(site));
1884 IFDBG(D_NONE, FN; NDBG(get_nodeno(site), u));
1885 IFDBG(D_NONE, FN; SYCEXP(site->start); SYCEXP(site->boot_key);
1886 NDBG(site->install_time, f));
1887 IFDBG(D_NONE, FN; NDBG(get_nodeno(site), u));
1888 ADD_DBG(
1889 D_BASE, add_event(EVENT_DUMP_PAD, string_arg("nodeno"));
1890 add_event(EVENT_DUMP_PAD, uint_arg(get_nodeno(site)));
1891 add_event(EVENT_DUMP_PAD, string_arg("site->boot_key"));
1892 add_synode_event(site->boot_key);
1893 /* add_event(EVENT_DUMP_PAD, uint_arg(chksum_node_list(&site->nodes))); */
1894 );
1895 12399 }
1896
1897 85 static void active_leaders(site_def *site, leader_array *leaders) {
1898 u_int i;
1899 u_int n;
1900 /* Synthesize leaders by copying all node names of active leaders */
1901
2/2
✓ Branch 0 taken 219 times.
✓ Branch 1 taken 85 times.
 304 for (i = 0, n = 0; i < site->nodes.node_list_len; i++) {
1902
2/2
✓ Branch 0 taken 157 times.
✓ Branch 1 taken 62 times.
 219 if (is_active_leader(i, site)) n++;
1903 }
1904 85 leaders->leader_array_len = n;
1905
1/2
✓ Branch 0 taken 85 times.
✗ Branch 1 not taken.
 85 if (n) {
1906 85 leaders->leader_array_val = static_cast<leader *>(
1907 85 xcom_calloc((size_t)leaders->leader_array_len, sizeof(leader)));
1908
2/2
✓ Branch 0 taken 219 times.
✓ Branch 1 taken 85 times.
 304 for (i = 0, n = 0; i < site->nodes.node_list_len; i++) {
1909
2/2
✓ Branch 0 taken 157 times.
✓ Branch 1 taken 62 times.
 219 if (is_active_leader(i, site)) {
1910 157 leaders->leader_array_val[n++].address =
1911 157 strdup(site->nodes.node_list_val[i].address);
1912 }
1913 }
1914 } else {
1915 ✗ leaders->leader_array_val = nullptr;
1916 }
1917 85 }
1918
1919 1 extern "C" void synthesize_leaders(leader_array *leaders) {
1920 // Default value meaning 'not set by client '
1921 1 leaders->leader_array_len = 0;
1922 1 leaders->leader_array_val = nullptr;
1923 1 }
1924
1925 145 static bool leaders_set_by_client(site_def const *site) {
1926 145 return site->leaders.leader_array_len != 0;
1927 }
1928
1929 1753 static site_def *create_site_def_with_start(app_data_ptr a, synode_no start) {
1930 1753 site_def *site = new_site_def();
1931 IFDBG(D_NONE, FN; COPY_AND_FREE_GOUT(dbg_list(&a->body.app_u_u.nodes)););
1932 1753 init_site_def(a->body.app_u_u.nodes.node_list_len,
1933 a->body.app_u_u.nodes.node_list_val, site);
1934 1753 site->start = start;
1935 1753 site->boot_key = a->app_key;
1936
1937 // If SINGLE_WRITER_ONLY is defined, ALL configs will be single writer. Used for
1938 // running all tests in single writer mode
1939 #ifdef SINGLE_WRITER_ONLY
1940 site->max_active_leaders = 1; /*Single writer */
1941 #else
1942 1753 site->max_active_leaders = active_leaders_all; /* Set to all nodes*/
1943 #endif
1944
1945 1753 return site;
1946 }
1947
1948 static xcom_proto constexpr single_writer_support = x_1_9;
1949
1950 1726 static site_def *install_ng_with_start(app_data_ptr a, synode_no start) {
1951
1/2
✓ Branch 0 taken 1726 times.
✗ Branch 1 not taken.
 1726 if (a) {
1952 1726 site_def *site = create_site_def_with_start(a, start);
1953 1726 site_def const *old_site = get_site_def();
1954
1955 // The reason why we need to recompute node sets and time stamps, is that
1956 // node sets and time stamps are stored in the site_def indexed by node
1957 // number, but they really are related to a specific node, not a specific
1958 // node number. When the site_def changes, the node number of a node may
1959 // change, thus invalidating the mapping from node numbers to node sets and
1960 // timestamps. But given the old and new definition, it is possible to
1961 // remap.
1962
3/4
✓ Branch 0 taken 878 times.
✓ Branch 1 taken 848 times.
✓ Branch 2 taken 878 times.
✗ Branch 3 not taken.
 1726 if (old_site && old_site->x_proto >= single_writer_support) {
1963 878 recompute_node_sets(old_site, site);
1964 878 recompute_timestamps(old_site->detected, &old_site->nodes, site->detected,
1965 878 &site->nodes);
1966 }
1967 1726 site_install_action(site, a->body.c_t);
1968 1726 return site;
1969 }
1970 ✗ return nullptr;
1971 }
1972
1973 1726 site_def *install_node_group(app_data_ptr a) {
1974 ADD_DBG(D_BASE, add_event(EVENT_DUMP_PAD, string_arg("a->app_key"));
1975 add_synode_event(a->app_key););
1976
1/2
✓ Branch 0 taken 1726 times.
✗ Branch 1 not taken.
 1726 if (a)
1977 1726 return install_ng_with_start(a, getstart(a));
1978 else
1979 ✗ return nullptr;
1980 }
1981
1982 192415 void set_max_synode(synode_no synode) {
1983 192415 max_synode = synode; /* Track max synode number */
1984 IFDBG(D_BASE, FN; STRLIT("new "); SYCEXP(max_synode));
1985 192415 activate_sweeper();
1986 192415 }
1987
1988 181500 static int is_busy(synode_no s) {
1989 181500 pax_machine *p = hash_get(s);
1990
2/2
✓ Branch 0 taken 67324 times.
✓ Branch 1 taken 114176 times.
 181500 if (!p) {
1991 67324 return 0;
1992 } else {
1993 114176 return started(p);
1994 }
1995 }
1996
1997 83493 bool_t match_my_msg(pax_msg *learned, pax_msg *mine) {
1998 IFDBG(D_NONE, FN; PTREXP(learned->a);
1999 if (learned->a) SYCEXP(learned->a->unique_id); PTREXP(mine->a);
2000 if (mine->a) SYCEXP(mine->a->unique_id););
2001
3/4
✓ Branch 0 taken 82146 times.
✓ Branch 1 taken 1347 times.
✓ Branch 2 taken 82146 times.
✗ Branch 3 not taken.
 83493 if (learned->a && mine->a) { /* Both have app data, see if data is mine */
2002 82146 return synode_eq(learned->a->unique_id, mine->a->unique_id);
2003
2/4
✓ Branch 0 taken 1347 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 1347 times.
 1347 } else if (!(learned->a || mine->a)) { /* None have app data, anything goes */
2004 ✗ return TRUE;
2005 } else { /* Definitely mismatch */
2006 1347 return FALSE;
2007 }
2008 }
2009
2010 /*
2011 * Initialize the log sequence number (lsn).
2012 */
2013 5638 void initialize_lsn(uint64_t n) { lsn = n; }
2014
2015 /**
2016 * Assign the next log sequence number (lsn) for a message.
2017 *
2018 * Initial propose sets lsn to msgno of the max message number as safe starting
2019 * point, otherwise lsn shall be ever increasing. lsn ensures sender order known
2020 * on receiver side, as messages may arrive "out of order" due to
2021 * retransmission. We use max_synode instead of current_message to avoid any
2022 * conflict with lsn allocated by a previous instance of the node.
2023 */
2024 85863 static uint64_t assign_lsn() {
2025
2/2
✓ Branch 0 taken 2215 times.
✓ Branch 1 taken 83648 times.
 85863 if (lsn == 0) {
2026 2215 initialize_lsn(max_synode.msgno);
2027 }
2028 85863 lsn++;
2029 IFDBG(D_EXEC, NDBG64(lsn));
2030 85863 return lsn;
2031 }
2032
2033 #if TASK_DBUG_ON
2034 /* purecov: begin deadcode */
2035 static int check_lsn(app_data_ptr a) {
2036 while (a) {
2037 if (!a->lsn) return 0;
2038 a = a->next;
2039 }
2040 return 1;
2041 }
2042 /* purecov: end */
2043 #endif
2044
2045 static void propose_noop(synode_no find, pax_machine *p);
2046
2047 /**
2048 * Checks if the given synod s is outside the event horizon.
2049 *
2050 * Common case: there are no configurations pending, or if there are, none of
2051 * them reconfigure the event horizon. The common case threshold is:
2052 *
2053 * last_executed_synod + event_horizon(active_config)
2054 *
2055 *
2056 * If an event horizon reconfiguration R is pending, it is possible that it
2057 * reduces the event horizon. In that case, it is possible that the threshold
2058 * above falls outside the new event horizon.
2059 *
2060 * For example, consider last_executed_synod = 42 and
2061 * event_horizon(active_config) = 10.
2062 * At this point this member participates in synods up to 52.
2063 * Now consider an event horizon reconfiguration that takes effect at synod 45,
2064 * which modifies the event horizon to 2. This means that when
2065 * last_executed_synod = 45, event_horizon(active_config) = 2. At this point
2066 * this member should only be able to participate in synods up to 47. The member
2067 * may have previously started processing messages directed to synods between 47
2068 * and 52, but will now ignore messages directed to those same synods.
2069 *
2070 * We do not want to start processing messages that will eventually fall out
2071 * of the event horizon. More importantly, the threshold above may not be safe
2072 * due to the exit logic of executor_task.
2073 *
2074 * When a node removes itself from the group on configuration C starting at
2075 * synod start(C), the exit logic relies on knowing *when* a majority has
2076 * executed synod start(C) - 1, i.e. the last message of the last configuration
2077 * to contain the leaving node.
2078 *
2079 * With a constant event horizon, we know that when synod
2080 * start(C) + event_horizon is learnt, it is because a majority already executed
2081 * or is ready to execute (and thus learned) synod start(C). This implies that a
2082 * majority already executed start(C) - 1.
2083 *
2084 * With a dynamic event horizon, we cannot be sure that when synod
2085 * start(C) + event_horizon(C) is learnt, a majority already executed or is
2086 * ready to execute synod start(C).
2087 * This is because it is possible for a new, smaller, event horizon to take
2088 * effect between start(C) and start(C) + event_horizon(C).
2089 * If that happens, the threshold above allows nodes to participate in synods
2090 * which are possibly beyond start(C) + event_horizon(C), which can lead to the
2091 * value of synod start(C) + event_horizon(C) being learnt without a majority
2092 * already having executed or being ready to execute synod start(C).
2093 *
2094 * In order to maintain the assumption made by the executor_task's exit logic,
2095 * when an event horizon reconfiguration R is pending we set the threshold to
2096 * the minimum between:
2097 *
2098 * last_executed_synod + event_horizon(active_config)
2099 *
2100 * and:
2101 *
2102 * start(R) - 1 + event_horizon(R)
2103 */
2104 1684197 static uint64_t too_far_threshold(xcom_event_horizon active_event_horizon) {
2105 1684197 return executed_msg.msgno + active_event_horizon;
2106 }
2107
2108 2687 static uint64_t too_far_threshold_new_event_horizon_pending(
2109 site_def const *new_config) {
2110 2687 uint64_t last_executed = executed_msg.msgno;
2111 /* compute normal threshold */
2112 uint64_t possibly_unsafe_threshold;
2113 2687 site_def const *active_config = find_site_def(executed_msg);
2114 2687 xcom_event_horizon active_event_horizon = active_config->event_horizon;
2115 2687 possibly_unsafe_threshold = last_executed + active_event_horizon;
2116 /* compute threshold taking into account new event horizon */ {
2117 uint64_t maximum_safe_threshold;
2118 xcom_event_horizon new_event_horizon;
2119 2687 uint64_t start_new_event_horizon = new_config->start.msgno;
2120 2687 new_event_horizon = new_config->event_horizon;
2121 2687 maximum_safe_threshold = start_new_event_horizon - 1 + new_event_horizon;
2122 /* use the minimum of both for safety */
2123
2/2
✓ Branch 0 taken 2532 times.
✓ Branch 1 taken 155 times.
 2687 return MIN(possibly_unsafe_threshold, maximum_safe_threshold);
2124 }
2125 }
2126
2127 1686884 static inline int too_far(synode_no s) {
2128 1686884 uint64_t threshold = 0;
2129 1686884 site_def const *active_config = find_site_def(executed_msg);
2130
1/2
✓ Branch 0 taken 1686884 times.
✗ Branch 1 not taken.
 1686884 if (active_config != nullptr) {
2131 1686884 site_def const *pending_config = first_event_horizon_reconfig();
2132 1686884 bool_t const no_event_horizon_reconfig_pending =
2133 1686884 (pending_config == nullptr);
2134
6/6
✓ Branch 0 taken 486118 times.
✓ Branch 1 taken 1200766 times.
✓ Branch 2 taken 483431 times.
✓ Branch 3 taken 2687 times.
✓ Branch 4 taken 1684197 times.
✓ Branch 5 taken 2687 times.
 1686884 if (is_latest_config(active_config) || no_event_horizon_reconfig_pending) {
2135 1684197 threshold = too_far_threshold(active_config->event_horizon);
2136 } else {
2137 2687 threshold = too_far_threshold_new_event_horizon_pending(pending_config);
2138 }
2139 } else {
2140 /* we have no configs, resort to default */
2141 ✗ threshold = too_far_threshold(EVENT_HORIZON_MIN);
2142 }
2143 1686884 return s.msgno >= threshold;
2144 }
2145
2146 #define GOTO(x) \
2147 { \
2148 IFDBG(D_NONE, STRLIT("goto "); STRLIT(#x)); \
2149 goto x; \
2150 }
2151
2152 81857 static inline int is_view(cargo_type x) { return x == view_msg; }
2153
2154 245801 static inline int is_config(cargo_type x) {
2155
4/4
✓ Branch 0 taken 241010 times.
✓ Branch 1 taken 3095 times.
✓ Branch 2 taken 234777 times.
✓ Branch 3 taken 6233 times.
 244105 return x == unified_boot_type || x == add_node_type ||
2156
4/4
✓ Branch 0 taken 234749 times.
✓ Branch 1 taken 28 times.
✓ Branch 2 taken 234679 times.
✓ Branch 3 taken 70 times.
 234777 x == remove_node_type || x == set_event_horizon_type ||
2157
5/6
✓ Branch 0 taken 244105 times.
✓ Branch 1 taken 1696 times.
✓ Branch 2 taken 234679 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 143 times.
✓ Branch 5 taken 234536 times.
 489906 x == force_config_type || x == set_max_leaders ||
2158 245801 x == set_leaders_type;
2159 }
2160
2161 static int wait_for_cache(pax_machine **pm, synode_no synode, double timeout);
2162 static int prop_started = 0;
2163 static int prop_finished = 0;
2164
2165 /* Find a free slot locally.
2166 Note that we will happily increment past the event horizon.
2167 The caller is thus responsible for checking the validity of the
2168 returned value by calling too_far() and ignore_message(). */
2169 84040 static synode_no local_synode_allocator(synode_no synode) {
2170
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 84040 times.
 84040 assert(!synode_eq(synode, null_synode));
2171
2172 // Ensure node number of synode is ours, whilst also ensuring that the synode
2173 // is monotonically increasing
2174 84040 node_no const my_nodeno = get_nodeno(find_site_def(synode));
2175
2/2
✓ Branch 0 taken 70041 times.
✓ Branch 1 taken 13999 times.
 84040 if (my_nodeno >= synode.node) {
2176 70041 synode.node = my_nodeno;
2177 } else {
2178 13999 synode = incr_msgno(synode);
2179 }
2180
2181
2/2
✓ Branch 0 taken 13446 times.
✓ Branch 1 taken 84040 times.
 97486 while (is_busy(synode)) {
2182 13446 synode = incr_msgno(synode);
2183 }
2184
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 84040 times.
 84040 assert(!synode_eq(synode, null_synode));
2185 84040 return synode;
2186 }
2187
2188 /* Find a likely free slot globally.
2189 Note that we will happily increment past the event horizon.
2190 The caller is thus responsible for checking the validity of the
2191 returned value by calling too_far() and ignore_message().
2192 The test for ignore_message() here is only valid until the
2193 event horizon. */
2194 ✗ static synode_no global_synode_allocator(site_def *site, synode_no synode) {
2195 ✗ assert(!synode_eq(synode, null_synode));
2196
2197 ✗ while (ignore_message(synode, site, "global_synode_allocator")) {
2198 ✗ synode = incr_synode(synode);
2199 }
2200 ✗ assert(!synode_eq(synode, null_synode));
2201 ✗ return synode;
2202 }
2203
2204 /* Find a free slot on remote leader */
2205 48681 static node_no remote_synode_allocator(site_def *site, app_data const &a) {
2206 static node_no distributor = 0; // Distribute requests equally among leaders
2207 48681 node_no maxnodes = get_maxnodes(site);
2208 48681 distributor = distributor % maxnodes; // Rescale in case site has changed
2209 48681 node_no i = distributor;
2210 // Ensure that current_message is associated with site
2211
2/2
✓ Branch 0 taken 29 times.
✓ Branch 1 taken 48652 times.
 48681 if (synode_lt(current_message, site->start)) {
2212 29 current_message = site->start;
2213 }
2214 for (;;) {
2215
4/4
✓ Branch 0 taken 49139 times.
✓ Branch 1 taken 194 times.
✓ Branch 2 taken 48014 times.
✓ Branch 3 taken 1319 times.
 98472 if (is_active_leader(i, site) &&
2216
2/2
✓ Branch 0 taken 48014 times.
✓ Branch 1 taken 1125 times.
 49139 !may_be_dead(site->detected, i,
2217 task_now())) { // Found leader, send request
2218 pax_msg *p =
2219 48014 pax_msg_new(current_message, site); // Message number does not matter
2220 IFDBG(D_CONS, FN; STRLIT("sending request "); NUMEXP(i);
2221 SYCEXP(current_message));
2222 48014 p->op = synode_request;
2223 48014 send_server_msg(site, i, p);
2224 48014 distributor = (i + 1) % maxnodes;
2225 48014 return i;
2226 }
2227 1319 i = (i + 1) % maxnodes;
2228
2/2
✓ Branch 0 taken 667 times.
✓ Branch 1 taken 652 times.
 1319 if (i == distributor) {
2229 /* There are no leaders, see if we should become leader. Note the special
2230 case for `force_config_type`. If we are in a network partition
2231 situation that must be healed using `force_config_type`, the leader
2232 might not be available and we might not be `iamthegreatest`. If we are
2233 the one tasked with `force_config_type` the entire system is relying on
2234 us to get consensus on `force_config_type` to unblock the group.
2235 Therefore, we self-allocate a synod for `force_config_type` to ensure
2236 the system makes progress. */
2237
3/6
✓ Branch 0 taken 667 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 667 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 667 times.
 667 if (iamthegreatest(site) || a.body.c_t == force_config_type) {
2238 // Grab message number and answer to self
2239 ✗ synode_no synode = global_synode_allocator(site, current_message);
2240 ✗ if (!too_far(synode)) {
2241 // We will grab this number, advance current_message
2242 ✗ set_current_message(incr_synode(synode));
2243 IFDBG(D_CONS, FN; STRLIT("grab message "); SYCEXP(synode);
2244 SYCEXP(current_message));
2245 ✗ synode_number_pool.put(synode, synode_allocation_type::global);
2246 }
2247 }
2248 667 return get_nodeno(site);
2249 }
2250 652 }
2251 }
2252
2253 #ifdef DELIVERY_TIMEOUT
2254 static bool check_delivery_timeout(site_def *site, double start_propose,
2255 app_data *a) {
2256 bool retval =
2257 (start_propose + a->expiry_time) < task_now() && !enough_live_nodes(site);
2258 if (retval) {
2259 DBGOUT_ASSERT(check_lsn(a), STRLIT("NULL lsn"));
2260 IFDBG(D_NONE, FN; STRLIT("timeout -> delivery_failure"));
2261 deliver_to_app(NULL, a, delivery_failure);
2262 }
2263 return retval;
2264 }
2265 #endif
2266
2267 154101 static int reserve_synode_number(synode_allocation_type *synode_allocation,
2268 site_def **site, synode_no *msgno,
2269 int *remote_retry, app_data *a,
2270 synode_reservation_status *ret) {
2271 154101 *ret = synode_reservation_status::number_ok; // Optimistic, will be reset if
2272 // necessary
2273 DECL_ENV
2274 int dummy;
2275 87210 ENV_INIT
2276 87210 END_ENV_INIT
2277 END_ENV;
2278
2279
6/11
✓ Branch 0 taken 87210 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 48681 times.
✓ Branch 3 taken 18210 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 87210 times.
✓ Branch 7 taken 87210 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 87210 times.
 154101 TASK_BEGIN
2280 do {
2281 87219 *synode_allocation = synode_allocation_type::todo;
2282 IFDBG(D_CONS, FN; SYCEXP(current_message));
2283 87219 *site = find_site_def_rw(current_message);
2284
2/2
✓ Branch 0 taken 83621 times.
✓ Branch 1 taken 3598 times.
 87219 if (is_leader(*site)) { // Use local synode allocator
2285 83621 *msgno = local_synode_allocator(current_message);
2286 IFDBG(D_CONS, FN; SYCEXP(outer_ep->msgno));
2287 83621 *synode_allocation = synode_allocation_type::local;
2288 } else { // Cannot use local, try remote
2289 // Get synode number from another leader
2290 3598 *remote_retry = 0;
2291
2/2
✓ Branch 0 taken 48681 times.
✓ Branch 1 taken 402 times.
 49083 while (synode_number_pool.empty()) {
2292 // get_maxnodes(get_site_def()) > 0 is a precondition for
2293 // `remote_synode_allocator`.
2294
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 48681 times.
 48681 if (get_maxnodes(get_site_def()) == 0) {
2295 ✗ TASK_DELAY(0.1);
2296 ✗ TASK_RETURN(synode_reservation_status::no_nodes);
2297 }
2298 #if TASK_DBUG_ON
2299 node_no allocator_node =
2300 #endif
2301 48681 remote_synode_allocator(get_site_def_rw(),
2302 *a); // Send request for synode, use
2303 // latest config
2304 48681 if (*remote_retry > 10) {
2305 IFDBG(D_BUG, FN; NUMEXP(outer_ep->self); NUMEXP(allocator_node);
2306 SYCEXP(executed_msg); SYCEXP(current_message);
2307 SYCEXP(outer_ep->msgno); SYCEXP(get_site_def_rw()->start));
2308 }
2309
1/2
✓ Branch 0 taken 48681 times.
✗ Branch 1 not taken.
 48681 if (synode_number_pool.empty()) { // Only wait if still empty
2310
4/6
✗ Branch 0 not taken.
✓ Branch 1 taken 48681 times.
✓ Branch 2 taken 48681 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 3196 times.
✓ Branch 5 taken 45485 times.
 97362 TIMED_TASK_WAIT(&synode_number_pool.queue,
2311 0.1); // Wait for incoming synode
2312 }
2313 45485 (*remote_retry)++;
2314 }
2315 402 std::tie(*msgno, *synode_allocation) = synode_number_pool.get();
2316 IFDBG(D_CONS, FN; SYCEXP(outer_ep->msgno));
2317 }
2318
2319 // Update site to match synode
2320 84023 *site = proposer_site = find_site_def_rw(*msgno);
2321
2322 // Set the global current message for all number allocators
2323 84023 set_current_message(incr_synode(*msgno));
2324
2325
2/2
✓ Branch 0 taken 18210 times.
✓ Branch 1 taken 84014 times.
 102224 while (too_far(*msgno)) { /* Too far ahead of executor */
2326
4/6
✗ Branch 0 not taken.
✓ Branch 1 taken 18210 times.
✓ Branch 2 taken 18210 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 9 times.
✓ Branch 5 taken 18201 times.
 36420 TIMED_TASK_WAIT(&exec_wait, 0.2);
2327 IFDBG(D_NONE, FN; SYCEXP(ep->msgno); TIMECEXP(ep->start_propose);
2328 TIMECEXP(outer_ep->client_msg->p->a->expiry_time);
2329 TIMECEXP(task_now()); NDBG(enough_live_nodes(outer_ep->site), d));
2330 #ifdef DELIVERY_TIMEOUT
2331 if (check_delivery_timeout(outer_ep->site, outer_ep->start_propose,
2332 outer_ep->client_msg->p->a)) {
2333 TASK_RETURN(delivery_timeout);
2334 }
2335 #endif
2336 }
2337 // Filter out busy or ignored message numbers
2338
6/6
✓ Branch 0 taken 84011 times.
✓ Branch 1 taken 3 times.
✓ Branch 2 taken 6 times.
✓ Branch 3 taken 84005 times.
✓ Branch 4 taken 9 times.
✓ Branch 5 taken 84005 times.
 84014 } while (is_busy(*msgno) || ignore_message(*msgno, *site, "proposer_task"));
2339 84005 FINALLY
2340
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 87210 times.
 87210 TASK_END;
2341 }
2342
2343 /* Send messages by fetching from the input queue and trying to get it accepted
2344 by a Paxos instance */
2345 281462 static int proposer_task(task_arg arg) {
2346 DECL_ENV
2347 int self; /* ID of this proposer task */
2348 pax_machine *p; /* Pointer to Paxos instance */
2349 msg_link *client_msg; /* The client message we are trying to push */
2350 synode_no msgno;
2351 pax_msg *prepare_msg;
2352 double start_propose;
2353 double start_push;
2354 double delay;
2355 site_def *site;
2356 size_t size;
2357 size_t nr_batched_app_data;
2358 int remote_retry;
2359 synode_allocation_type synode_allocation;
2360 22150 ENV_INIT
2361 22150 END_ENV_INIT
2362 END_ENV;
2363
2364 281462 synode_reservation_status reservation_status{
2365 synode_reservation_status::number_ok};
2366
2367
9/17
✓ Branch 0 taken 22150 times.
✓ Branch 1 taken 103712 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 66891 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 88709 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 22150 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 22150 times.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✓ Branch 12 taken 22150 times.
✓ Branch 13 taken 22150 times.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✓ Branch 16 taken 22150 times.
 281462 TASK_BEGIN
2368
2369 22150 ep->self = get_int_arg(arg);
2370 22150 ep->p = nullptr;
2371 22150 ep->client_msg = nullptr;
2372 22150 ep->prepare_msg = nullptr;
2373 22150 ep->start_propose = 0.0;
2374 22150 ep->start_push = 0.0;
2375 22150 ep->delay = 0.0;
2376 22150 ep->msgno = current_message;
2377 22150 ep->site = nullptr;
2378 22150 ep->size = 0;
2379 22150 ep->nr_batched_app_data = 0;
2380 22150 ep->remote_retry = 0;
2381 22150 ep->synode_allocation = synode_allocation_type::todo;
2382 22150 add_proposer_synode(ep->self, &ep->msgno);
2383 IFDBG(D_NONE, FN; NDBG(ep->self, d); NDBG(task_now(), f));
2384
2385
1/2
✓ Branch 0 taken 104607 times.
✗ Branch 1 not taken.
 104607 while (!xcom_shutdown) { /* Loop until no more work to do */
2386 /* Wait for client message */
2387
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 104607 times.
 104607 assert(!ep->client_msg);
2388
8/12
✓ Branch 0 taken 103782 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 103712 times.
✓ Branch 4 taken 103712 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 18674 times.
✓ Branch 7 taken 85038 times.
✓ Branch 8 taken 103782 times.
✓ Branch 9 taken 85863 times.
✓ Branch 10 taken 85863 times.
✗ Branch 11 not taken.
 208319 CHANNEL_GET(&prop_input_queue, &ep->client_msg, msg_link);
2389 85863 prop_started++;
2390 IFDBG(D_NONE, FN; PTREXP(ep->client_msg->p->a); STRLIT("extracted ");
2391 SYCEXP(ep->client_msg->p->a->app_key));
2392
2393 /* Grab rest of messages in queue as well, but never batch config messages,
2394 * which need a unique number */
2395
2396 /* The batch is limited either by size or number of batched app_datas.
2397 * We limit the number of elements because the XDR deserialization
2398 * implementation is recursive, and batching too many app_datas will cause a
2399 * call stack overflow. */
2400
6/6
✓ Branch 0 taken 81367 times.
✓ Branch 1 taken 4496 times.
✓ Branch 2 taken 75003 times.
✓ Branch 3 taken 6364 times.
✓ Branch 4 taken 75003 times.
✓ Branch 5 taken 10860 times.
 167230 if (!is_config(ep->client_msg->p->a->body.c_t) &&
2401 81367 !is_view(ep->client_msg->p->a->body.c_t)) {
2402
1/2
✓ Branch 0 taken 75003 times.
✗ Branch 1 not taken.
 75003 ep->size = app_data_size(ep->client_msg->p->a);
2403 75003 ep->nr_batched_app_data = 1;
2404 75003 while (AUTOBATCH && ep->size <= MAX_BATCH_SIZE &&
2405
6/8
✓ Branch 0 taken 75473 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 75473 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 505 times.
✓ Branch 5 taken 74968 times.
✓ Branch 6 taken 505 times.
✓ Branch 7 taken 74968 times.
 150946 ep->nr_batched_app_data <= MAX_BATCH_APP_DATA &&
2406 75473 !link_empty(&prop_input_queue
2407 .data)) { /* Batch payloads into single message */
2408 msg_link *tmp;
2409 app_data_ptr atmp;
2410
2411
2/12
✗ Branch 0 not taken.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✓ Branch 9 taken 505 times.
✓ Branch 10 taken 505 times.
✗ Branch 11 not taken.
 505 CHANNEL_GET(&prop_input_queue, &tmp, msg_link);
2412 505 atmp = tmp->p->a;
2413
1/2
✓ Branch 0 taken 505 times.
✗ Branch 1 not taken.
 505 ep->size += app_data_size(atmp);
2414 505 ep->nr_batched_app_data++;
2415 /* Abort batching if config or too big batch */
2416
2/2
✓ Branch 0 taken 470 times.
✓ Branch 1 taken 20 times.
 995 if (is_config(atmp->body.c_t) || is_view(atmp->body.c_t) ||
2417
5/6
✓ Branch 0 taken 490 times.
✓ Branch 1 taken 15 times.
✓ Branch 2 taken 470 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 35 times.
✓ Branch 5 taken 470 times.
 1465 ep->nr_batched_app_data > MAX_BATCH_APP_DATA ||
2418
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 470 times.
 470 ep->size > MAX_BATCH_SIZE) {
2419
1/2
✓ Branch 0 taken 35 times.
✗ Branch 1 not taken.
 35 channel_put_front(&prop_input_queue, &tmp->l);
2420 35 break;
2421 }
2422 ADD_T_EV(seconds(), __FILE__, __LINE__, "batching");
2423
2424 470 tmp->p->a = nullptr; /* Steal this payload */
2425
1/2
✓ Branch 0 taken 470 times.
✗ Branch 1 not taken.
 470 msg_link_delete(&tmp); /* Get rid of the empty message */
2426 470 atmp->next = ep->client_msg->p->a; /* Add to list of app_data */
2427 /* G_TRACE("Batching %s %s",
2428 * cargo_type_to_str(ep->client_msg->p->a->body.c_t), */
2429 /* cargo_type_to_str(atmp->body.c_t)); */
2430 470 ep->client_msg->p->a = atmp;
2431 IFDBG(D_NONE, FN; PTREXP(ep->client_msg->p->a); STRLIT("extracted ");
2432 SYCEXP(ep->client_msg->p->a->app_key));
2433 }
2434 }
2435
2436
1/2
✓ Branch 0 taken 85863 times.
✗ Branch 1 not taken.
 85863 ep->start_propose = task_now();
2437 85863 ep->delay = 0.0;
2438
2439
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 85863 times.
 85863 assert(!ep->client_msg->p->a->chosen);
2440
2441 /* It is a new message */
2442
2443
2/4
✓ Branch 0 taken 85863 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 85863 times.
 85863 assert(!synode_eq(current_message, null_synode));
2444
2445 /* Assign a log sequence number only on initial propose */
2446 {
2447
1/2
✓ Branch 0 taken 85863 times.
✗ Branch 1 not taken.
 85863 uint64_t prop_lsn = assign_lsn();
2448 85863 app_data_ptr ap = ep->client_msg->p->a;
2449 /* Assign to all app_data structs */
2450
2/2
✓ Branch 0 taken 86372 times.
✓ Branch 1 taken 85863 times.
 172235 while (ap) {
2451 86372 ap->lsn = prop_lsn;
2452 86372 ap = ap->next;
2453 }
2454 }
2455 DBGOUT_ASSERT(check_lsn(ep->client_msg->p->a), STRLIT("NULL lsn"));
2456 85863 retry_new:
2457 /* Find a free slot */
2458
12/18
✓ Branch 0 taken 87210 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 154101 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 87210 times.
✓ Branch 5 taken 66891 times.
✓ Branch 6 taken 3205 times.
✓ Branch 7 taken 84005 times.
✓ Branch 8 taken 66891 times.
✓ Branch 9 taken 84005 times.
✗ Branch 10 not taken.
✓ Branch 11 taken 66891 times.
✗ Branch 12 not taken.
✓ Branch 13 taken 66891 times.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✓ Branch 16 taken 66891 times.
✓ Branch 17 taken 84005 times.
 220992 TASK_CALL(reserve_synode_number(&ep->synode_allocation, &ep->site,
2459 &ep->msgno, &ep->remote_retry,
2460 ep->client_msg->p->a, &reservation_status));
2461
2462 // Check result of reservation
2463
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 84005 times.
 84005 if (reservation_status == synode_reservation_status::no_nodes) {
2464 ✗ GOTO(retry_new);
2465
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 84005 times.
 84005 } else if (reservation_status ==
2466 synode_reservation_status::delivery_timeout) {
2467 ✗ GOTO(next);
2468 }
2469 // If we get here, we have a valid synode number
2470
2/4
✓ Branch 0 taken 84005 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 84005 times.
 84005 assert(!synode_eq(ep->msgno, null_synode));
2471
2472 /* See if we can do anything with this message */
2473
6/8
✓ Branch 0 taken 84005 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 84005 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 311 times.
✓ Branch 5 taken 83694 times.
✓ Branch 6 taken 311 times.
✓ Branch 7 taken 83694 times.
 84005 if (!ep->site || get_nodeno(ep->site) == VOID_NODE_NO) {
2474 /* Give up */
2475 DBGOUT_ASSERT(check_lsn(ep->client_msg->p->a), STRLIT("NULL lsn"));
2476 IFDBG(D_NONE, FN; STRLIT("delivery_failure "); SYCEXP(ep->msgno);
2477 PTREXP(ep->site); NDBG(get_nodeno(ep->site), u));
2478
1/2
✓ Branch 0 taken 311 times.
✗ Branch 1 not taken.
 311 deliver_to_app(nullptr, ep->client_msg->p->a, delivery_failure);
2479 311 GOTO(next);
2480 }
2481
2482
1/2
✓ Branch 0 taken 83694 times.
✗ Branch 1 not taken.
 83694 brand_client_msg(ep->client_msg->p, ep->msgno);
2483
2484 for (;;) { /* Loop until the client message has been learned */
2485 /* Get a Paxos instance to send the client message */
2486
2487
6/18
✓ Branch 0 taken 83694 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 83694 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 83694 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 83694 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 83694 times.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✗ Branch 16 not taken.
✓ Branch 17 taken 83694 times.
 83694 TASK_CALL(wait_for_cache(&ep->p, ep->msgno, 60));
2488
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 83694 times.
 83694 if (!ep->p) {
2489 ✗ G_MESSAGE("Could not get a pax_machine for msgno %lu. Retrying",
2490 (unsigned long)ep->msgno.msgno);
2491 ✗ goto retry_new;
2492 }
2493
2494
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 83694 times.
 83694 assert(ep->p);
2495
2/2
✓ Branch 0 taken 27 times.
✓ Branch 1 taken 83667 times.
 83694 if (ep->client_msg->p->force_delivery)
2496 27 ep->p->force_delivery = ep->client_msg->p->force_delivery;
2497 {
2498
1/2
✓ Branch 0 taken 83694 times.
✗ Branch 1 not taken.
 83694 [[maybe_unused]] int lock = lock_pax_machine(ep->p);
2499
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 83694 times.
 83694 assert(!lock);
2500 }
2501
2502 /* Set the client message as current proposal */
2503
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 83694 times.
 83694 assert(ep->client_msg->p);
2504
2/4
✓ Branch 0 taken 83694 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 83694 times.
✗ Branch 3 not taken.
 83694 replace_pax_msg(&ep->p->proposer.msg, clone_pax_msg(ep->client_msg->p));
2505
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 83694 times.
 83694 if (ep->p->proposer.msg == nullptr) {
2506 ✗ g_critical(
2507 "Node %u has run out of memory while sending a message and "
2508 "will now exit.",
2509 get_nodeno(proposer_site));
2510 ✗ terminate_and_exit(); /* Tell xcom to stop */
2511 ✗ TERMINATE;
2512 }
2513
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 83694 times.
 83694 assert(ep->p->proposer.msg);
2514 PAX_MSG_SANITY_CHECK(ep->p->proposer.msg);
2515
2516 /* Create the prepare message */
2517
1/2
✓ Branch 0 taken 83694 times.
✗ Branch 1 not taken.
 83694 unchecked_replace_pax_msg(&ep->prepare_msg,
2518
1/2
✓ Branch 0 taken 83694 times.
✗ Branch 1 not taken.
 83694 pax_msg_new(ep->msgno, ep->site));
2519 IFDBG(D_NONE, FN; PTREXP(ep->client_msg->p->a); STRLIT("pushing ");
2520 SYCEXP(ep->msgno));
2521 IFDBG(D_NONE, FN; COPY_AND_FREE_GOUT(dbg_app_data(ep->prepare_msg->a)));
2522
2523 /* Use 3 phase algorithm if threephase is set or we are forcing or we have
2524 already accepted something, which may happen if another node has timed
2525 out waiting for this node and proposed a no_op, which we have accepted.
2526
2527 We *must* use 3 phase algorithm if the synode was allocated by
2528 ourselves using `global_synode_allocator`. This is last resort synode
2529 allocation that does not guarantee we will be the only Proposer using
2530 it. Therefore, for correctness we must use regular 3 phase Paxos,
2531 because we may have dueling Proposers.
2532 */
2533
3/4
✓ Branch 0 taken 83638 times.
✓ Branch 1 taken 56 times.
✓ Branch 2 taken 83638 times.
✗ Branch 3 not taken.
 83694 if (threephase || ep->p->force_delivery || ep->p->acceptor.promise.cnt ||
2534
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 83638 times.
 83638 ep->synode_allocation == synode_allocation_type::global) {
2535
1/2
✓ Branch 0 taken 56 times.
✗ Branch 1 not taken.
 56 push_msg_3p(ep->site, ep->p, ep->prepare_msg, ep->msgno, normal);
2536 } else {
2537
1/2
✓ Branch 0 taken 83638 times.
✗ Branch 1 not taken.
 83638 push_msg_2p(ep->site, ep->p);
2538 }
2539
2540
1/2
✓ Branch 0 taken 83694 times.
✗ Branch 1 not taken.
 83694 ep->start_push = task_now();
2541
2542
1/2
✓ Branch 0 taken 88709 times.
✗ Branch 1 not taken.
 88709 while (!finished(ep->p)) { /* Try to get a value accepted */
2543 /* We will wake up periodically, and whenever a message arrives */
2544
8/14
✓ Branch 0 taken 88709 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 88709 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 88709 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 88709 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✓ Branch 9 taken 88709 times.
✓ Branch 10 taken 88709 times.
✗ Branch 11 not taken.
✓ Branch 12 taken 201 times.
✓ Branch 13 taken 88508 times.
 177418 TIMED_TASK_WAIT(&ep->p->rv, ep->delay = wakeup_delay(ep->delay));
2545
3/6
✓ Branch 0 taken 88508 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 88508 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 88508 times.
 177016 if (!synode_eq(ep->msgno, ep->p->synode) ||
2546
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 88508 times.
 88508 ep->p->proposer.msg == nullptr) {
2547 IFDBG(D_NONE, FN; STRLIT("detected stolen state machine, retry"););
2548 /* unlock_pax_machine(ep->p); */
2549 ✗ GOTO(retry_new); /* Need to break out of both loops,
2550 and we have no "exit named
2551 loop" construction */
2552 }
2553
3/6
✓ Branch 0 taken 88508 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 88508 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 88508 times.
✗ Branch 5 not taken.
 88508 assert(synode_eq(ep->msgno, ep->p->synode) && ep->p->proposer.msg);
2554
2/2
✓ Branch 0 taken 83493 times.
✓ Branch 1 taken 5015 times.
 88508 if (finished(ep->p)) break;
2555 {
2556
1/2
✓ Branch 0 taken 5015 times.
✗ Branch 1 not taken.
 5015 double now = task_now();
2557 #ifdef DELIVERY_TIMEOUT
2558 if ((ep->start_propose + ep->client_msg->p->a->expiry_time) < now) {
2559 IFDBG(D_NONE, FN; STRLIT("timeout when pushing ");
2560 SYCEXP(ep->msgno); SYCEXP(executed_msg));
2561 /* Proposing a no-op here is a last ditch effort to cancel the
2562 failed message. If any of the currently reachable nodes have
2563 participated in the failed consensus round, it is equivalent to
2564 retrying a final time, otherwise we could get a no-op
2565 accepted. Proposing a no-op is always harmless.
2566 Having a timeout on delivery and telling the client is really
2567 contrary to the spirit of
2568 Paxos, since we cannot guarantee that the message has not been
2569 delivered, but at the moment, MCM depends on it.
2570 Proposing a no-op here increases the probability that the outcome
2571 matches what we tell MCM about the outcome. */
2572 propose_noop(ep->msgno, ep->p);
2573 DBGOUT_ASSERT(check_lsn(ep->client_msg->p->a), STRLIT("NULL lsn"));
2574 IFDBG(D_NONE, FN; STRLIT("timeout -> delivery_failure"));
2575 deliver_to_app(ep->p, ep->client_msg->p->a, delivery_failure);
2576 unlock_pax_machine(ep->p);
2577 GOTO(next);
2578 }
2579 #endif
2580
2/2
✓ Branch 0 taken 2633 times.
✓ Branch 1 taken 2382 times.
 5015 if ((ep->start_push + ep->delay) <= now) {
2581 PAX_MSG_SANITY_CHECK(ep->p->proposer.msg);
2582 IFDBG(D_NONE, FN; STRLIT("retry pushing "); SYCEXP(ep->msgno));
2583 IFDBG(D_NONE, FN;
2584 COPY_AND_FREE_GOUT(dbg_app_data(ep->prepare_msg->a)););
2585 IFDBG(D_NONE, BALCEXP(ep->p->proposer.bal);
2586 BALCEXP(ep->p->acceptor.promise));
2587
1/2
✓ Branch 0 taken 2633 times.
✗ Branch 1 not taken.
 2633 push_msg_3p(ep->site, ep->p, ep->prepare_msg, ep->msgno, normal);
2588 2633 ep->start_push = now;
2589 }
2590 }
2591 }
2592 /* When we get here, we know the value for this message number,
2593 but it may not be the value we tried to push,
2594 so loop until we have a successful push. */
2595
1/2
✓ Branch 0 taken 83493 times.
✗ Branch 1 not taken.
 83493 unlock_pax_machine(ep->p);
2596 IFDBG(D_NONE, FN; STRLIT(" found finished message "); SYCEXP(ep->msgno);
2597 STRLIT("seconds since last push ");
2598 NPUT(task_now() - ep->start_push, f); STRLIT("ep->client_msg ");
2599 COPY_AND_FREE_GOUT(dbg_pax_msg(ep->client_msg->p)););
2600 IFDBG(D_NONE, FN; STRLIT("ep->p->learner.msg ");
2601 COPY_AND_FREE_GOUT(dbg_pax_msg(ep->p->learner.msg)););
2602
3/4
✓ Branch 0 taken 83493 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 82146 times.
✓ Branch 3 taken 1347 times.
 83493 if (match_my_msg(ep->p->learner.msg, ep->client_msg->p)) {
2603 82146 break;
2604 } else
2605 1347 GOTO(retry_new);
2606 }
2607 82457 next : {
2608
1/2
✓ Branch 0 taken 82457 times.
✗ Branch 1 not taken.
 82457 double now = task_now();
2609 82457 double used = now - ep->start_propose;
2610
1/2
✓ Branch 0 taken 82457 times.
✗ Branch 1 not taken.
 82457 add_to_filter(used);
2611 82457 prop_finished++;
2612 IFDBG(D_NONE, FN; STRLIT("completed ep->msgno "); SYCEXP(ep->msgno);
2613 NDBG(used, f); NDBG(median_time(), f);
2614 STRLIT("seconds since last push "); NDBG(now - ep->start_push, f););
2615 IFDBG(D_NONE, FN; STRLIT("ep->client_msg ");
2616 COPY_AND_FREE_GOUT(dbg_pax_msg(ep->client_msg->p)););
2617 82457 if (ep->p) {
2618 IFDBG(D_NONE, FN; STRLIT("ep->p->learner.msg ");
2619 COPY_AND_FREE_GOUT(dbg_pax_msg(ep->p->learner.msg)););
2620 }
2621
1/2
✓ Branch 0 taken 82457 times.
✗ Branch 1 not taken.
 82457 msg_link_delete(&ep->client_msg);
2622 }
2623 }
2624 ✗ FINALLY
2625 IFDBG(D_BUG, FN; STRLIT("exit "); NDBG(ep->self, d); NDBG(task_now(), f));
2626
2/2
✓ Branch 0 taken 19962 times.
✓ Branch 1 taken 2118 times.
 22080 if (ep->p) {
2627
1/2
✓ Branch 0 taken 19962 times.
✗ Branch 1 not taken.
 19962 unlock_pax_machine(ep->p);
2628 }
2629
1/2
✓ Branch 0 taken 22080 times.
✗ Branch 1 not taken.
 22080 replace_pax_msg(&ep->prepare_msg, nullptr);
2630
2/2
✓ Branch 0 taken 3406 times.
✓ Branch 1 taken 18674 times.
 22080 if (ep->client_msg) { /* If we get here with a client message, we have
2631 failed to deliver */
2632 DBGOUT_ASSERT(check_lsn(ep->client_msg->p->a), STRLIT("NULL lsn"));
2633 IFDBG(D_NONE, FN;
2634 STRLIT("undelivered message at task end -> delivery_failure"));
2635
1/2
✓ Branch 0 taken 3406 times.
✗ Branch 1 not taken.
 3406 deliver_to_app(ep->p, ep->client_msg->p->a, delivery_failure);
2636
1/2
✓ Branch 0 taken 3406 times.
✗ Branch 1 not taken.
 3406 msg_link_delete(&ep->client_msg);
2637 }
2638 22080 remove_proposer_synode(ep->self);
2639
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 22080 times.
✓ Branch 2 taken 22080 times.
✗ Branch 3 not taken.
 22080 TASK_END;
2640 }
2641
2642 static xcom_proto constexpr first_protocol_that_ignores_intermediate_forced_configs_or_views =
2643 x_1_8;
2644
2645 ✗ static bool constexpr should_ignore_forced_config_or_view(
2646 xcom_proto protocol_version) {
2647 ✗ return protocol_version >=
2648 ✗ first_protocol_that_ignores_intermediate_forced_configs_or_views;
2649 }
2650
2651 52241 static node_no get_leader(site_def const *s) {
2652
1/2
✓ Branch 0 taken 52241 times.
✗ Branch 1 not taken.
 52241 if (s) {
2653 52241 node_no leader = 0;
2654
2/2
✓ Branch 0 taken 54334 times.
✓ Branch 1 taken 671 times.
 55005 for (leader = 0; leader < get_maxnodes(s); leader++) {
2655
2/2
✓ Branch 0 taken 51570 times.
✓ Branch 1 taken 2764 times.
 54334 if (!may_be_dead(s->detected, leader, task_now())) return leader;
2656 }
2657 }
2658 671 return 0;
2659 }
2660
2661 52241 int iamthegreatest(site_def const *s) {
2662
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 52241 times.
 52241 if (!s)
2663 ✗ return 0;
2664 else
2665 52241 return get_leader(s) == s->nodeno;
2666 }
2667
2668 // Update site based on incoming global node set
2669 51 static site_def *update_site(site_def *site, node_set const *ns,
2670 synode_no boot_key, synode_no start) {
2671 // If it has not changed, no action is necessary.
2672 // If it has changed, we need to create and install
2673 // a new site def, since the changed node set will influence which
2674 // messages will be ignored.
2675 // This change needs to be effective after the current pipeline
2676 // of messages has been emptied, just as if we had
2677 // changed the config (site_def) itself.
2678
2679
2/2
✓ Branch 0 taken 12 times.
✓ Branch 1 taken 39 times.
 51 if (!equal_node_set(ns, &site->global_node_set)) {
2680 12 site_def *new_config = clone_site_def(get_site_def());
2681
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 12 times.
 12 assert(new_config);
2682 12 new_config->start = start;
2683 12 new_config->boot_key = boot_key;
2684 // Update node set of site
2685 12 copy_node_set(ns, &new_config->global_node_set);
2686 12 return new_config;
2687 }
2688 39 return nullptr;
2689 }
2690
2691 159361 void execute_msg(site_def *site, pax_machine *pma, pax_msg *p) {
2692 159361 app_data_ptr a = p->a;
2693 IFDBG(D_EXEC, FN; COPY_AND_FREE_GOUT(dbg_pax_msg(p)););
2694
1/2
✓ Branch 0 taken 159361 times.
✗ Branch 1 not taken.
 159361 if (a) {
2695
5/5
✓ Branch 0 taken 878 times.
✓ Branch 1 taken 5753 times.
✓ Branch 2 taken 141338 times.
✓ Branch 3 taken 11269 times.
✓ Branch 4 taken 123 times.
 159361 switch (a->body.c_t) {
2696 878 case unified_boot_type:
2697 case force_config_type:
2698 878 deliver_config(a);
2699 6631 case add_node_type:
2700 case remove_node_type:
2701 6631 break;
2702 141338 case app_type:
2703 IFDBG(D_NONE, FN; STRLIT(" learner.msg ");
2704 COPY_AND_FREE_GOUT(dbg_pax_msg(pma->learner.msg)););
2705 /* DBGOUT_ASSERT(check_lsn(a), STRLIT("NULL lsn")); */
2706 141338 deliver_to_app(pma, a, delivery_ok);
2707 141338 break;
2708 11269 case view_msg: {
2709 /* Deliver view like we used to when every member was always a leader.
2710 * This ensures deterministic behaviour in groups with some members
2711 * running previous XCom instances.
2712 */
2713 IFDBG(D_EXEC, FN; STRLIT(" global view ");
2714 COPY_AND_FREE_GOUT(dbg_pax_msg(pma->learner.msg)););
2715
1/2
✓ Branch 0 taken 11269 times.
✗ Branch 1 not taken.
 11269 if (site && site->global_node_set.node_set_len ==
2716
1/2
✓ Branch 0 taken 11269 times.
✗ Branch 1 not taken.
 11269 a->body.app_u_u.present.node_set_len) {
2717
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 11269 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 11269 times.
 11269 if ((p->force_delivery != 0) &&
2718 ✗ should_ignore_forced_config_or_view(site->x_proto)) {
2719 ✗ G_DEBUG(
2720 "execute_msg: Ignoring a forced intermediate, pending "
2721 "view_msg");
2722 } else {
2723
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11269 times.
 11269 assert(site->global_node_set.node_set_len ==
2724 a->body.app_u_u.present.node_set_len);
2725 // Can only mutate site->global_node_set if everyone is a leader and
2726 // has its own channel.
2727
2/2
✓ Branch 0 taken 11212 times.
✓ Branch 1 taken 57 times.
 11269 if (site->max_active_leaders == active_leaders_all) {
2728 11212 copy_node_set(&a->body.app_u_u.present, &site->global_node_set);
2729 }
2730 11269 deliver_global_view_msg(site, a->body.app_u_u.present, p->synode);
2731 ADD_DBG(D_BASE,
2732 add_event(EVENT_DUMP_PAD,
2733 string_arg("deliver_global_view_msg p->synode"));
2734 add_synode_event(p->synode););
2735 }
2736 }
2737
2738 /* If this view_msg is:
2739 *
2740 * (1) about the latest site, and
2741 * (2) only some member(s) is (are) leader(s) in the latest site,
2742 *
2743 * create a new site to deterministically ignore the channel of leaders
2744 * that may be dead. */
2745 11269 site_def *latest_site = get_site_def_rw();
2746 IFDBG(
2747 D_EXEC, FN; PTREXP(latest_site); if (latest_site) {
2748 NUMEXP(latest_site->nodes.node_list_len);
2749 NUMEXP(latest_site->global_node_set.node_set_len);
2750 NUMEXP(a->body.app_u_u.present.node_set_len);
2751 SYCEXP(a->app_key);
2752 SYCEXP(latest_site->start);
2753 });
2754 /*
2755 * You'll want to install the new site if xcom is operating as
2756 * single-leader and there were no changes in the configuration. The
2757 * reason for this is so that you have the latest information about
2758 * who is the preferred and actual leader.
2759 */
2760 11269 bool const is_latest_view = synode_gt(a->app_key, latest_site->start);
2761 11269 bool const everyone_leader_in_latest_site =
2762 11269 (latest_site->max_active_leaders == active_leaders_all);
2763 11269 bool const view_node_set_matches_latest_site =
2764 11269 (latest_site->global_node_set.node_set_len ==
2765 11269 a->body.app_u_u.present.node_set_len);
2766 22515 const bool can_install_site = is_latest_view &&
2767
5/6
✓ Branch 0 taken 11246 times.
✓ Branch 1 taken 23 times.
✓ Branch 2 taken 51 times.
✓ Branch 3 taken 11195 times.
✓ Branch 4 taken 51 times.
✗ Branch 5 not taken.
 11269 !everyone_leader_in_latest_site &&
2768 view_node_set_matches_latest_site;
2769
2770
2/2
✓ Branch 0 taken 51 times.
✓ Branch 1 taken 11218 times.
 11269 if (can_install_site) {
2771 51 a->app_key = p->synode; // Patch app_key to avoid fixing getstart()
2772 51 site_def *new_config = update_site(
2773 51 latest_site, &a->body.app_u_u.present, a->app_key, getstart(a));
2774
2/2
✓ Branch 0 taken 12 times.
✓ Branch 1 taken 39 times.
 51 if (new_config) {
2775 IFDBG(D_EXEC, FN; PTREXP(new_config);
2776 NUMEXP(new_config->nodes.node_list_len);
2777 NUMEXP(new_config->global_node_set.node_set_len);
2778 SYCEXP(a->app_key); SYCEXP(new_config->start););
2779 12 site_install_action(new_config, a->body.c_t);
2780 12 analyze_leaders(new_config);
2781 }
2782 }
2783 11269 } break;
2784 123 default:
2785 123 break;
2786 }
2787 }
2788 IFDBG(D_NONE, FN; SYCEXP(p->synode));
2789 159361 }
2790
2791 static void read_missing_values(int n);
2792 static void propose_missing_values(int n);
2793
2794 #ifdef EXECUTOR_TASK_AGGRESSIVE_NO_OP
2795 /* With many nodes sending read_ops on instances that are not decided yet, it
2796 * may take a very long time until someone finally decides to start a new
2797 * consensus round. As the cost of a new proposal is not that great, it's
2798 * acceptable to go directly to proposing a no-op instead of first trying to
2799 * get the value with a read_op. An added benefit of this is that if more than
2800 * one node needs the result, they will get it all when the consensus round
2801 * finishes. */
2802 static void find_value(site_def const *site, unsigned int *wait, int n) {
2803 IFDBG(D_NONE, FN; NDBG(*wait, d));
2804
2805 if (get_nodeno(site) == VOID_NODE_NO) {
2806 read_missing_values(n);
2807 return;
2808 }
2809
2810 if ((*wait) > 1 || /* Only leader will propose initially */
2811 ((*wait) > 0 && iamthegreatest(site)))
2812 propose_missing_values(n);
2813
2814 #ifdef TASK_EVENT_TRACE
2815 if ((*wait) > 1) dump_task_events();
2816 #endif
2817 (*wait)++;
2818 }
2819 #else
2820 360097 static void find_value(site_def const *site, unsigned int *wait, int n) {
2821 IFDBG(D_NONE, FN; NDBG(*wait, d));
2822
2823
2/2
✓ Branch 0 taken 6926 times.
✓ Branch 1 taken 353171 times.
 360097 if (get_nodeno(site) == VOID_NODE_NO) {
2824 6926 read_missing_values(n);
2825 6926 return;
2826 }
2827
2828
3/4
✓ Branch 0 taken 253303 times.
✓ Branch 1 taken 51574 times.
✓ Branch 2 taken 48294 times.
✗ Branch 3 not taken.
 353171 switch (*wait) {
2829 253303 case 0:
2830 case 1:
2831 253303 read_missing_values(n);
2832 253303 (*wait)++;
2833 253303 break;
2834 51574 case 2:
2835
2/2
✓ Branch 0 taken 33068 times.
✓ Branch 1 taken 18506 times.
 51574 if (iamthegreatest(site))
2836 33068 propose_missing_values(n);
2837 else
2838 18506 read_missing_values(n);
2839 51574 (*wait)++;
2840 51574 break;
2841 48294 case 3:
2842 48294 propose_missing_values(n);
2843 48294 break;
2844 ✗ default:
2845 ✗ break;
2846 }
2847 }
2848 #endif /* EXECUTOR_TASK_AGGRESSIVE_NO_OP */
2849
2850 static void dump_debug_exec_state();
2851
2852 #ifdef PROPOSE_IF_LEADER
2853 int get_xcom_message(pax_machine **p, synode_no msgno, int n) {
2854 DECL_ENV
2855 unsigned int wait;
2856 double delay;
2857 site_def const *site;
2858 ENV_INIT
2859 END_ENV_INIT
2860 END_ENV;
2861
2862 TASK_BEGIN
2863
2864 ep->wait = 0;
2865 ep->delay = 0.0;
2866 *p = force_get_cache(msgno);
2867 ep->site = NULL;
2868
2869 dump_debug_exec_state();
2870 while (!finished(*p)) {
2871 ep->site = find_site_def(msgno);
2872 /* The end of the world ?, fake message by skipping */
2873 if (get_maxnodes(ep->site) == 0) {
2874 pax_msg *msg = pax_msg_new(msgno, ep->site);
2875 handle_skip(ep->site, *p, msg);
2876 break;
2877 }
2878 IFDBG(D_NONE, FN; STRLIT(" not finished "); SYCEXP(msgno); PTREXP(*p);
2879 NDBG(ep->wait, u); SYCEXP(msgno));
2880 if (get_maxnodes(ep->site) > 1 && iamthegreatest(ep->site) &&
2881 ep->site->global_node_set.node_set_val &&
2882 !ep->site->global_node_set.node_set_val[msgno.node] &&
2883 may_be_dead(ep->site->detected, msgno.node, task_now())) {
2884 propose_missing_values(n);
2885 } else {
2886 find_value(ep->site, &ep->wait, n);
2887 }
2888 TIMED_TASK_WAIT(&(*p)->rv, ep->delay = wakeup_delay(ep->delay));
2889 *p = get_cache(msgno);
2890 dump_debug_exec_state();
2891 }
2892
2893 FINALLY
2894 IFDBG(D_NONE, FN; SYCEXP(msgno); PTREXP(*p); NDBG(ep->wait, u);
2895 SYCEXP(msgno));
2896 TASK_END;
2897 }
2898 #else
2899 887387 int get_xcom_message(pax_machine **p, synode_no msgno, int n) {
2900 DECL_ENV
2901 unsigned int wait;
2902 double delay;
2903 site_def const *site;
2904 527297 ENV_INIT
2905 527297 END_ENV_INIT
2906 END_ENV;
2907
2908
5/9
✓ Branch 0 taken 527297 times.
✓ Branch 1 taken 360090 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 527297 times.
✓ Branch 5 taken 527297 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 527297 times.
 887387 TASK_BEGIN
2909
2910 527297 ep->wait = 0;
2911 527297 ep->delay = 0.0;
2912 527297 *p = force_get_cache(msgno);
2913 527297 ep->site = nullptr;
2914
2915 527297 dump_debug_exec_state();
2916
2/2
✓ Branch 0 taken 360097 times.
✓ Branch 1 taken 527269 times.
 887366 while (!finished(*p)) {
2917 360097 ep->site = find_site_def(msgno);
2918 /* The end of the world ?, fake message by skipping */
2919
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 360097 times.
 360097 if (get_maxnodes(ep->site) == 0) {
2920 ✗ pax_msg *msg = pax_msg_new(msgno, ep->site);
2921 ✗ handle_skip(ep->site, *p, msg);
2922 ✗ break;
2923 }
2924 IFDBG(D_NONE, FN; STRLIT("before find_value"); SYCEXP(msgno); PTREXP(*p);
2925 NDBG(ep->wait, u); SYCEXP(msgno));
2926 360097 find_value(ep->site, &ep->wait, n);
2927 IFDBG(D_NONE, FN; STRLIT("after find_value"); SYCEXP(msgno); PTREXP(*p);
2928 NDBG(ep->wait, u); SYCEXP(msgno));
2929 360097 ep->delay = wakeup_delay(ep->delay);
2930 IFDBG(D_NONE, FN; NDBG(ep->delay, f));
2931
4/6
✗ Branch 0 not taken.
✓ Branch 1 taken 360090 times.
✓ Branch 2 taken 360090 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 21 times.
✓ Branch 5 taken 360069 times.
 720187 TIMED_TASK_WAIT(&(*p)->rv, ep->delay);
2932 360069 *p = get_cache(msgno);
2933 360069 dump_debug_exec_state();
2934 }
2935
2936 527269 FINALLY
2937
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 527290 times.
 527290 TASK_END;
2938 }
2939 #endif
2940
2941 532599 synode_no set_executed_msg(synode_no msgno) {
2942 IFDBG(D_EXEC, FN; STRLIT("changing executed_msg from "); SYCEXP(executed_msg);
2943 STRLIT(" to "); SYCEXP(msgno));
2944
3/4
✓ Branch 0 taken 532599 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 189072 times.
✓ Branch 3 taken 343527 times.
 1065198 if (group_mismatch(msgno, current_message) ||
2945
2/2
✓ Branch 0 taken 189072 times.
✓ Branch 1 taken 343527 times.
 532599 synode_gt(msgno, current_message)) {
2946 IFDBG(D_EXEC, FN; STRLIT("changing current message"));
2947 189072 set_current_message(first_free_synode_local(msgno));
2948 }
2949
2950
2/2
✓ Branch 0 taken 272184 times.
✓ Branch 1 taken 260415 times.
 532599 if (msgno.msgno > executed_msg.msgno) task_wakeup(&exec_wait);
2951
2952 532599 executed_msg = msgno;
2953 532599 executor_site = find_site_def_rw(executed_msg);
2954 532599 return executed_msg;
2955 }
2956
2957 190439 static synode_no first_free_synode_local(synode_no msgno) {
2958
1/2
✓ Branch 0 taken 190439 times.
✗ Branch 1 not taken.
 190439 site_def const *site = find_site_def(msgno);
2959 190439 synode_no retval = msgno;
2960
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 190439 times.
 190439 if (!site) {
2961 /* purecov: begin deadcode */
2962 ✗ site = get_site_def();
2963 IFDBG(D_NONE, FN; PTREXP(site); SYCEXP(msgno));
2964 ✗ assert(get_group_id(site) != 0);
2965 /* purecov: end */
2966 }
2967
1/2
✓ Branch 0 taken 190439 times.
✗ Branch 1 not taken.
 190439 if (get_group_id(site) == 0) {
2968 IFDBG(D_NONE, FN; PTREXP(site); SYCEXP(msgno));
2969 if (site) {
2970 IFDBG(D_NONE, FN; SYCEXP(site->boot_key); SYCEXP(site->start);
2971 COPY_AND_FREE_GOUT(dbg_site_def(site)));
2972 }
2973 }
2974
2/4
✓ Branch 0 taken 190439 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 190439 times.
 190439 assert(get_group_id(site) != 0);
2975
2/4
✓ Branch 0 taken 190439 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 190439 times.
 190439 assert(!synode_eq(msgno, null_synode));
2976
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 190439 times.
 190439 if (retval.msgno == 0) retval.msgno = 1;
2977
1/2
✓ Branch 0 taken 190439 times.
✗ Branch 1 not taken.
 190439 retval.node = get_nodeno(site);
2978
3/4
✓ Branch 0 taken 190439 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 55238 times.
✓ Branch 3 taken 135201 times.
 190439 if (synode_lt(retval, msgno))
2979
1/2
✓ Branch 0 taken 55238 times.
✗ Branch 1 not taken.
 55238 return incr_msgno(retval);
2980 else
2981 135201 return retval;
2982 }
2983
2984 274860 synode_no set_current_message(synode_no msgno) {
2985 IFDBG(D_PROPOSE, FN; STRLIT("changing current_message from ");
2986 SYCEXP(current_message); STRLIT(" to "); SYCEXP(msgno));
2987 274860 return current_message = msgno;
2988 }
2989
2990 static void update_max_synode(pax_msg *p);
2991
2992 #if TASK_DBUG_ON
2993 static void perf_dbg(int *_n, int *_old_n, double *_old_t) [[maybe_unused]];
2994 static void perf_dbg(int *_n, int *_old_n, double *_old_t) {
2995 int n = *_n;
2996 int old_n = *_old_n;
2997 double old_t = *_old_t;
2998
2999 if (!IS_XCOM_DEBUG_WITH(XCOM_DEBUG_TRACE)) return;
3000
3001 IFDBG(D_NONE, FN; SYCEXP(executed_msg));
3002 if (!(n % 5000)) {
3003 GET_GOUT;
3004 NDBG(get_nodeno(get_site_def()), u);
3005 NDBG(task_now(), f);
3006 NDBG(n, d);
3007 NDBG(median_time(), f);
3008 SYCEXP(executed_msg);
3009 PRINT_GOUT;
3010 FREE_GOUT;
3011 }
3012 (*_n)++;
3013 if (task_now() - old_t > 1.0) {
3014 GET_GOUT;
3015 NDBG(get_nodeno(get_site_def()), u);
3016 NDBG(task_now(), f);
3017 NDBG(n, d);
3018 NDBG((n - old_n) / (task_now() - old_t), f);
3019 PRINT_GOUT;
3020 FREE_GOUT;
3021 *_old_t = task_now();
3022 *_old_n = n;
3023 }
3024 }
3025 #endif
3026
3027 /* Does address match any current leader? */
3028 74 static inline int match_leader(char const *addr, leader_array const leaders) {
3029 u_int i;
3030
2/2
✓ Branch 0 taken 74 times.
✓ Branch 1 taken 20 times.
 94 for (i = 0; i < leaders.leader_array_len; i++) {
3031 IFDBG(D_BASE, FN; NUMEXP(i); NUMEXP(leaders.leader_array_len); STREXP(addr);
3032 STREXP(leaders.leader_array_val[i].address));
3033
2/2
✓ Branch 0 taken 54 times.
✓ Branch 1 taken 20 times.
 74 if (strcmp(addr, leaders.leader_array_val[i].address) == 0) return 1;
3034 }
3035 20 return 0;
3036 }
3037
3038 82 static inline bool alive_node(site_def const *site, u_int i) {
3039 82 return is_set(site->global_node_set, i);
3040 }
3041
3042 // Find up to site->max_active_leaders leaders.
3043 // If leaders are set by the client, and none of those are alive, revert
3044 // to using the set of addresses in the config.
3045
3046 59 void analyze_leaders(site_def *site) {
3047
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 59 times.
 59 assert(site);
3048 // No analysis if all nodes are leaders
3049
2/2
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 58 times.
 59 if (active_leaders_all == site->max_active_leaders) return;
3050
3051 // Use leaders from config if forced or not set by client
3052 58 bool use_client_leaders = leaders_set_by_client(site);
3053 58 site->cached_leaders = true;
3054 58 site->found_leaders = 0; // Number of active leaders found
3055 // Reset everything
3056
2/2
✓ Branch 0 taken 125 times.
✓ Branch 1 taken 58 times.
 183 for (u_int i = 0; i < get_maxnodes(site); i++) {
3057 125 site->active_leader[i] = 0;
3058 }
3059 // If candidate leaders set by client, check those first
3060
6/6
✓ Branch 0 taken 179 times.
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 123 times.
✓ Branch 3 taken 56 times.
✓ Branch 4 taken 123 times.
✓ Branch 5 taken 58 times.
 181 for (u_int i = 0; use_client_leaders && i < get_maxnodes(site); i++) {
3061 246 if (site->found_leaders <
3062 200 site->max_active_leaders && // Found enough?
3063 // Must be alive according to global
3064 // node set of site
3065
8/8
✓ Branch 0 taken 77 times.
✓ Branch 1 taken 46 times.
✓ Branch 2 taken 74 times.
✓ Branch 3 taken 3 times.
✓ Branch 4 taken 54 times.
✓ Branch 5 taken 20 times.
✓ Branch 6 taken 54 times.
✓ Branch 7 taken 69 times.
 197 alive_node(site, i) &&
3066 // Must match a node in the list of leaders
3067 74 match_leader(site->nodes.node_list_val[i].address, site->leaders)) {
3068 54 site->active_leader[i] = 1;
3069 54 site->found_leaders++;
3070 }
3071 }
3072 // Check rest of nodes
3073
2/2
✓ Branch 0 taken 125 times.
✓ Branch 1 taken 58 times.
 183 for (u_int i = 0; i < get_maxnodes(site); i++) {
3074 321 if (!site->active_leader[i] && // Avoid duplicates
3075
6/6
✓ Branch 0 taken 71 times.
✓ Branch 1 taken 54 times.
✓ Branch 2 taken 5 times.
✓ Branch 3 taken 66 times.
✓ Branch 4 taken 2 times.
✓ Branch 5 taken 123 times.
 130 site->found_leaders < site->max_active_leaders && // Found enough?
3076 // Must be alive according to global
3077 // node set of site
3078
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 3 times.
 5 alive_node(site, i)) {
3079 2 site->active_leader[i] = 1;
3080 2 site->found_leaders++;
3081 }
3082 }
3083 // We need at least one channel otherwise the group grinds to a halt.
3084
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 56 times.
 58 if (site->found_leaders == 0) {
3085 2 site->active_leader[0] = 1;
3086 2 site->found_leaders = 1;
3087 }
3088 58 free(site->dispatch_table);
3089
3090 IFDBG(D_BUG, FN; STRLIT("free "); PTREXP(site); PTREXP(site->dispatch_table));
3091 // Do not work as synode allocator if not active leader. ???
3092
4/4
✓ Branch 0 taken 49 times.
✓ Branch 1 taken 9 times.
✓ Branch 2 taken 25 times.
✓ Branch 3 taken 33 times.
 107 if (get_nodeno(site) != VOID_NODE_NO &&
3093
2/2
✓ Branch 0 taken 25 times.
✓ Branch 1 taken 24 times.
 49 site->active_leader[get_nodeno(site)]) {
3094 25 site->dispatch_table = primary_dispatch_table();
3095 } else {
3096 33 site->dispatch_table = secondary_dispatch_table();
3097 }
3098 IFDBG(D_BUG, FN; STRLIT("allocate "); PTREXP(site);
3099 PTREXP(site->dispatch_table));
3100
3101
2/2
✓ Branch 0 taken 125 times.
✓ Branch 1 taken 58 times.
 183 for (u_int i = 0; i < get_maxnodes(site); i++) {
3102 IFDBG(D_BUG, FN; NUMEXP(i); PTREXP(site); NUMEXP(site->found_leaders);
3103 NUMEXP(site->max_active_leaders); NUMEXP(alive_node(site, i));
3104 SYCEXP(site->start); STREXP(site->nodes.node_list_val[i].address);
3105 if (site->active_leader[i]) STRLIT(" says YES");
3106 else STRLIT(" says NO"));
3107 }
3108 }
3109
3110 /* Is node number an active leader? */
3111 1474355 int is_active_leader(node_no x, site_def *site) {
3112 /* No site, no active leaders */
3113
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1474355 times.
 1474355 if (!site) return 0;
3114
3115 /* Node number out of bound, not an active leader */
3116
2/2
✓ Branch 0 taken 6730 times.
✓ Branch 1 taken 1467625 times.
 1474355 if (x >= get_maxnodes(site)) return 0;
3117
3118 /* All are leaders, no need for further tests */
3119
2/2
✓ Branch 0 taken 1460301 times.
✓ Branch 1 taken 7324 times.
 1467625 if (active_leaders_all == site->max_active_leaders) return 1;
3120 /* See if cached values are valid */
3121
2/2
✓ Branch 0 taken 41 times.
✓ Branch 1 taken 7283 times.
 7324 if (!site->cached_leaders) {
3122 41 analyze_leaders(site);
3123 }
3124 #if 0
3125 if (site->active_leader == NULL || x > site->nodes.node_list_len - 1)
3126 IFDBG(D_BUG, FN; PTREXP(site->active_leader); NUMEXP(x);
3127 NUMEXP(site->nodeno); NUMEXP(site->nodes.node_list_len););
3128 #endif
3129 7324 return site->active_leader[x];
3130 }
3131
3132 6 node_no found_active_leaders(site_def *site) {
3133 /* No site, no active leaders */
3134
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6 times.
 6 if (!site) return 0;
3135
3136 /* All are leaders, no need for further tests */
3137
2/2
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 5 times.
 6 if (active_leaders_all == site->max_active_leaders)
3138 1 return site->nodes.node_list_len;
3139
3140 /* See if cached values are valid */
3141
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5 times.
 5 if (!site->cached_leaders) {
3142 ✗ analyze_leaders(site);
3143 }
3144 5 return site->found_leaders;
3145 }
3146
3147 /* Check if this message belongs to a channel that should be ignored */
3148 1336937 static inline int ignore_message(synode_no x, site_def *site,
3149 char const *dbg [[maybe_unused]]) {
3150 1336937 int retval = !is_active_leader(x.node, site);
3151 IFDBG(D_BASE, STRLIT(dbg); STRLIT(" "); FN; SYCEXP(x); NUMEXP(retval));
3152 1336937 return retval;
3153 }
3154
3155 /* Check if this node is a leader */
3156 87639 static inline bool is_leader(site_def *site) {
3157
3/4
✓ Branch 0 taken 87639 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 84040 times.
✓ Branch 3 taken 3599 times.
 87639 bool retval = site && is_active_leader(site->nodeno, site);
3158 IFDBG(D_BASE, FN; PTREXP(site); if (site) NUMEXP(site->nodeno);
3159 NUMEXP(retval));
3160 87639 return retval;
3161 }
3162
3163 [[maybe_unused]] static void debug_loser(synode_no x);
3164 #if defined(TASK_DBUG_ON) && TASK_DBUG_ON
3165 static void debug_loser(synode_no x) {
3166 if (!IS_XCOM_DEBUG_WITH(XCOM_DEBUG_TRACE)) return;
3167 if (1 || x.msgno < 10) {
3168 GET_GOUT;
3169 NDBG(get_nodeno(find_site_def(x)), u);
3170 STRLIT(" ignoring loser ");
3171 SYCEXP(x);
3172 SYCEXP(max_synode);
3173 PRINT_GOUT;
3174 FREE_GOUT;
3175 }
3176 }
3177 #else
3178 /* purecov: begin deadcode */
3179 ✗ static void debug_loser(synode_no x [[maybe_unused]]) {}
3180 /* purecov: end */
3181 #endif
3182
3183 33104 static void send_value(site_def const *site, node_no to, synode_no synode) {
3184 33104 pax_machine *pm = get_cache(synode);
3185
3/4
✓ Branch 0 taken 33104 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 30317 times.
✓ Branch 3 taken 2787 times.
 33104 if (pm && pm->learner.msg) {
3186
1/2
✓ Branch 0 taken 30317 times.
✗ Branch 1 not taken.
 30317 pax_msg *msg = clone_pax_msg(pm->learner.msg);
3187
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 30317 times.
 30317 if (msg == nullptr) return;
3188
1/2
✓ Branch 0 taken 30317 times.
✗ Branch 1 not taken.
 30317 ref_msg(msg);
3189
1/2
✓ Branch 0 taken 30317 times.
✗ Branch 1 not taken.
 30317 send_server_msg(site, to, msg);
3190
1/2
✓ Branch 0 taken 30317 times.
✗ Branch 1 not taken.
 30317 unref_msg(&msg);
3191 }
3192 }
3193
3194 /**
3195 * Returns the message number where it is safe for nodes in previous
3196 * configuration to exit.
3197 *
3198 * @param start start synod of the next configuration
3199 * @param event_horizon event horizon of the next configuration
3200 */
3201 8500 static synode_no compute_delay(synode_no start,
3202 xcom_event_horizon event_horizon) {
3203 8500 start.msgno += event_horizon;
3204 8500 return start;
3205 }
3206
3207 /* Push messages to all nodes which were in the previous site, but not in this
3208 */
3209 8911 static void inform_removed(int index, int all) {
3210 8911 site_def **sites = nullptr;
3211 8911 uint32_t site_count = 0;
3212 IFDBG(D_NONE, FN; NEXP(index, d));
3213
1/2
✓ Branch 0 taken 8911 times.
✗ Branch 1 not taken.
 8911 get_all_site_defs(&sites, &site_count);
3214
4/6
✓ Branch 0 taken 8911 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 6724 times.
✓ Branch 3 taken 2187 times.
✓ Branch 4 taken 6724 times.
✗ Branch 5 not taken.
 8911 while (site_count > 1 && index >= 0 && (uint32_t)(index + 1) < site_count) {
3215 6724 site_def *s = sites[index];
3216 6724 site_def *ps = sites[index + 1];
3217
3218 /* Compute diff and push messages */
3219 IFDBG(D_NONE, FN; NDBG(index, d); PTREXP(s); if (s) SYCEXP(s->boot_key);
3220 PTREXP(ps); if (ps) SYCEXP(ps->boot_key));
3221
3222
2/4
✓ Branch 0 taken 6724 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 6724 times.
✗ Branch 3 not taken.
 6724 if (s && ps) {
3223 6724 node_no i = 0;
3224 IFDBG(D_NONE, FN; SYCEXP(s->boot_key); SYCEXP(s->start);
3225 SYCEXP(ps->boot_key); SYCEXP(ps->start));
3226
2/2
✓ Branch 0 taken 12426 times.
✓ Branch 1 taken 6724 times.
 19150 for (i = 0; i < ps->nodes.node_list_len; i++) { /* Loop over prev site */
3227
4/4
✓ Branch 0 taken 5702 times.
✓ Branch 1 taken 6724 times.
✓ Branch 2 taken 1802 times.
✓ Branch 3 taken 10624 times.
 18128 if (ps->nodeno != i &&
3228
3/4
✓ Branch 0 taken 5702 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1802 times.
✓ Branch 3 taken 3900 times.
 5702 !node_exists(&ps->nodes.node_list_val[i], &s->nodes)) {
3229 1802 synode_no synode = s->start;
3230 1802 synode_no end = max_synode;
3231
3/4
✓ Branch 0 taken 34906 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 33104 times.
✓ Branch 3 taken 1802 times.
 34906 while (!synode_gt(synode, end)) { /* Loop over relevant messages */
3232
1/2
✓ Branch 0 taken 33104 times.
✗ Branch 1 not taken.
 33104 send_value(ps, i, synode);
3233
1/2
✓ Branch 0 taken 33104 times.
✗ Branch 1 not taken.
 33104 synode = incr_synode(synode);
3234 }
3235 }
3236 }
3237 }
3238
1/2
✓ Branch 0 taken 6724 times.
✗ Branch 1 not taken.
 6724 if (!all) /* Early exit if not all configs should be examined */
3239 6724 break;
3240 ✗ index--;
3241 }
3242 8911 }
3243
3244 ✗ static bool_t backwards_compatible(xcom_event_horizon event_horizon) {
3245 ✗ return event_horizon == EVENT_HORIZON_MIN;
3246 }
3247
3248 static xcom_proto const first_event_horizon_aware_protocol = x_1_4;
3249
3250 3521 static bool_t reconfigurable_event_horizon(xcom_proto protocol_version) {
3251 3521 return protocol_version >= first_event_horizon_aware_protocol;
3252 }
3253
3254 1729 static bool_t add_node_unsafe_against_ipv4_old_nodes(app_data_ptr a) {
3255
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1729 times.
 1729 assert(a->body.c_t == add_node_type);
3256
3257 {
3258 1729 site_def const *latest_config = get_site_def();
3259
3/6
✓ Branch 0 taken 1729 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1729 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1729 times.
✗ Branch 5 not taken.
 1729 if (latest_config && latest_config->x_proto >= minimum_ipv6_version())
3260 1729 return FALSE;
3261
3262 {
3263 ✗ u_int const nr_nodes_to_add = a->body.app_u_u.nodes.node_list_len;
3264 ✗ node_address *nodes_to_add = a->body.app_u_u.nodes.node_list_val;
3265
3266 u_int i;
3267 ✗ xcom_port node_port = 0;
3268 char node_addr[IP_MAX_SIZE];
3269
3270 ✗ for (i = 0; i < nr_nodes_to_add; i++) {
3271 ✗ if (get_ip_and_port(nodes_to_add[i].address, node_addr, &node_port)) {
3272 ✗ G_ERROR(
3273 "Error parsing address from a joining node. Join operation "
3274 "will be "
3275 "rejected");
3276 ✗ return TRUE;
3277 }
3278
3279 ✗ if (!is_node_v4_reachable(node_addr)) return TRUE;
3280 }
3281 }
3282
3283 ✗ return FALSE;
3284 }
3285 }
3286
3287 /**
3288 * @brief This will test if we are receiving a boot request that contains
3289 * ourselves. This could happed in case of a misconfiguration of a
3290 * local_address, that causes an add_node request to be erroneous
3291 * delivered.
3292 *
3293 * @param a app_data with an add node request
3294 * @return bool_t TRUE in case of error, meaning that my address is in the
3295 * add_node list
3296 */
3297 ✗ static bool_t add_node_adding_own_address(app_data_ptr a) {
3298 ✗ assert(a->body.c_t == add_node_type);
3299
3300 ✗ return node_exists(cfg_app_xcom_get_identity(), &a->body.app_u_u.nodes);
3301 }
3302
3303 /**
3304 * Check if a node is compatible with the group's event horizon.
3305 *
3306 * A node is compatible with the group's configuration if:
3307 *
3308 * a) The node supports event horizon reconfigurations, or
3309 * b) The group's event horizon is, or is scheduled to be, the default
3310 * event horizon.
3311 */
3312 3493 static bool unsafe_against_event_horizon(node_address const *node) {
3313 3493 site_def const *latest_config = get_site_def();
3314 3493 xcom_proto node_max_protocol_version = node->proto.max_proto;
3315 bool const compatible =
3316
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 3493 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 3493 reconfigurable_event_horizon(node_max_protocol_version) ||
3317 ✗ backwards_compatible(latest_config->event_horizon);
3318
3319
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3493 times.
 3493 if (!compatible) {
3320 /*
3321 * The node that wants to join does not support event horizon
3322 * reconfigurations and the group's event horizon is, or is scheduled to
3323 * be, different from the default.
3324 * The node can not safely join the group so we deny its attempt to join.
3325 */
3326 ✗ G_INFO(
3327 "%s's request to join the group was rejected because the group's "
3328 "event "
3329 "horizon is, or will be %" PRIu32 " and %s only supports %" PRIu32,
3330 node->address, latest_config->event_horizon, node->address,
3331 EVENT_HORIZON_MIN);
3332 ✗ return true;
3333 }
3334 3493 return false;
3335 }
3336
3337 typedef bool (*unsafe_node_check)(node_address const *node);
3338
3339 10495 static bool check_if_add_node_is_unsafe(app_data_ptr a,
3340 unsafe_node_check unsafe) {
3341
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 10495 times.
 10495 assert(a->body.c_t == add_node_type);
3342 {
3343 10495 u_int nodes_len = a->body.app_u_u.nodes.node_list_len;
3344 10495 node_address *nodes_to_add = a->body.app_u_u.nodes.node_list_val;
3345 u_int i;
3346
2/2
✓ Branch 0 taken 10493 times.
✓ Branch 1 taken 10493 times.
 20986 for (i = 0; i < nodes_len; i++) {
3347
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 10491 times.
 10493 if (unsafe(&nodes_to_add[i])) return true;
3348 }
3349 }
3350 10493 return false;
3351 }
3352
3353 3493 static bool check_if_add_node_is_unsafe_against_event_horizon(app_data_ptr a) {
3354 3493 return check_if_add_node_is_unsafe(a, unsafe_against_event_horizon);
3355 }
3356
3357 // Map values of old node set to new node set by matching on
3358 // node addresses
3359 13265 void recompute_node_set(node_set const *old_set, node_list const *old_nodes,
3360 node_set *new_set, node_list const *new_nodes) {
3361 // Return value of node set matching node_address na
3362 19798 auto value{[&](node_address const *na) {
3363
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 19798 times.
 19798 assert(old_set->node_set_len == old_nodes->node_list_len);
3364
2/2
✓ Branch 0 taken 31346 times.
✓ Branch 1 taken 3528 times.
 34874 for (u_int i = 0; i < old_nodes->node_list_len; i++) {
3365
2/2
✓ Branch 0 taken 16270 times.
✓ Branch 1 taken 15076 times.
 31346 if (match_node(&old_nodes->node_list_val[i], na, true)) {
3366 16270 return old_set->node_set_val[i];
3367 }
3368 }
3369 3528 return 0;
3370 13265 }};
3371
3372
2/2
✓ Branch 0 taken 19798 times.
✓ Branch 1 taken 13265 times.
 33063 for (u_int i = 0; i < new_nodes->node_list_len; i++) {
3373
1/2
✓ Branch 0 taken 19798 times.
✗ Branch 1 not taken.
 19798 new_set->node_set_val[i] = value(&new_nodes->node_list_val[i]);
3374 }
3375 13265 }
3376
3377 // Remap old global and local node set of site to new
3378 6631 static void recompute_node_sets(site_def const *old_site, site_def *new_site) {
3379 6631 recompute_node_set(&old_site->global_node_set, &old_site->nodes,
3380 6631 &new_site->global_node_set, &new_site->nodes);
3381 6631 recompute_node_set(&old_site->local_node_set, &old_site->nodes,
3382 6631 &new_site->local_node_set, &new_site->nodes);
3383 6631 }
3384
3385 3651 static bool incompatible_proto_and_max_leaders(xcom_proto x_proto,
3386 node_no max_leaders) {
3387
4/4
✓ Branch 0 taken 7 times.
✓ Branch 1 taken 3644 times.
✓ Branch 2 taken 4 times.
✓ Branch 3 taken 3 times.
 3651 return x_proto < single_writer_support && max_leaders != active_leaders_all;
3388 }
3389
3390 3659 static bool incompatible_proto_and_leaders(xcom_proto x_proto) {
3391 3659 return x_proto < single_writer_support;
3392 }
3393
3394 3501 static bool incompatible_proto_and_max_leaders(node_address const *node) {
3395 3501 site_def const *latest_config = get_site_def();
3396
3397
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 3499 times.
 3501 if (incompatible_proto_and_max_leaders(node->proto.max_proto,
3398 3501 latest_config->max_active_leaders)) {
3399 /*
3400 * The node that wants to join does not allow setting of max number of
3401 * leaders
3402 * and the max number of leaders in the group is not all.
3403 * The node can not safely join the group so we deny its attempt to join.
3404 */
3405
2/4
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 G_INFO(
3406 "%s's request to join the group was rejected because the group's max "
3407 "number of active leaders is, or will be %" PRIu32
3408 " and %s only supports "
3409 "all nodes as leaders",
3410 node->address, latest_config->max_active_leaders, node->address);
3411 2 return true;
3412 }
3413 3499 return false;
3414 }
3415
3416 3499 static bool incompatible_proto_and_leaders(node_address const *node) {
3417 3499 site_def const *latest_config = get_site_def();
3418
3419
4/6
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 3497 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 3499 times.
 3501 if (incompatible_proto_and_leaders(node->proto.max_proto) &&
3420 2 leaders_set_by_client(latest_config)) {
3421 /*
3422 * The node that wants to join does not allow changing the set of
3423 * leaders
3424 * and the set of leaders in the group is not empty.
3425 * The node can not safely join the group so we deny its attempt to join.
3426 */
3427 ✗ G_INFO(
3428 "%s's request to join the group was rejected because the group "
3429 "has a non-empty set of leaders specified by the client, "
3430 "and %s does not support changing the set of leaders",
3431 node->address, node->address);
3432 ✗ return true;
3433 }
3434 3499 return false;
3435 }
3436
3437 3502 bool unsafe_leaders(app_data *a) {
3438
2/2
✓ Branch 0 taken 3500 times.
✓ Branch 1 taken 2 times.
 7002 return check_if_add_node_is_unsafe(a, incompatible_proto_and_max_leaders) ||
3439
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3500 times.
 7002 check_if_add_node_is_unsafe(a, incompatible_proto_and_leaders);
3440 }
3441
3442 5989 static void set_start_and_boot(site_def *new_config, app_data_ptr a) {
3443 5989 new_config->start = getstart(a);
3444 5989 new_config->boot_key = a->app_key;
3445 5989 }
3446
3447 // Map values of old timestamps to new timestamps by matching on
3448 // node addresses
3449 6634 void recompute_timestamps(detector_state const old_timestamp,
3450 node_list const *old_nodes,
3451 detector_state new_timestamp,
3452 node_list const *new_nodes) {
3453 // Return value of timestamp matching node_address na
3454 9901 auto value{[&](node_address const *na) {
3455
2/2
✓ Branch 0 taken 15676 times.
✓ Branch 1 taken 1764 times.
 17440 for (u_int i = 0; i < old_nodes->node_list_len; i++) {
3456
2/2
✓ Branch 0 taken 8137 times.
✓ Branch 1 taken 7539 times.
 15676 if (match_node(&old_nodes->node_list_val[i], na, true)) {
3457 8137 return old_timestamp[i];
3458 }
3459 }
3460 1764 return 0.0;
3461 6634 }};
3462
3463
2/2
✓ Branch 0 taken 9901 times.
✓ Branch 1 taken 6634 times.
 16535 for (u_int i = 0; i < new_nodes->node_list_len; i++) {
3464
1/2
✓ Branch 0 taken 9901 times.
✗ Branch 1 not taken.
 9901 new_timestamp[i] = value(&new_nodes->node_list_val[i]);
3465 }
3466 6634 }
3467
3468 /**
3469 * Reconfigure the group membership: add new member(s).
3470 *
3471 * It is possible that concurrent reconfigurations take effect between the
3472 * time this reconfiguration was proposed and now.
3473 *
3474 * Particularly, it is possible that any of the concurrent reconfigurations
3475 * modified the event horizon and that the new member(s) do not support event
3476 * horizon reconfigurations.
3477 *
3478 * We account for these situations by validating if adding the new members is
3479 * still possible under the current state.
3480 *
3481 * If it is not, this reconfiguration does not produce any effect, i.e. no new
3482 * configuration is installed.
3483 */
3484 1764 site_def *handle_add_node(app_data_ptr a) {
3485
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1764 times.
 1764 if (check_if_add_node_is_unsafe_against_event_horizon(a)) {
3486 /*
3487 * Note that the result of this function is only applicable to
3488 * unused and not-fully-implemented code paths where add_node_type is used
3489 * forcibly.
3490 * Should this fact change, this obviously does not work.
3491 */
3492 ✗ return nullptr;
3493 }
3494
3495
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1764 times.
 1764 if (unsafe_leaders(a)) {
3496 ✗ return nullptr;
3497 }
3498 {
3499
2/2
✓ Branch 0 taken 1764 times.
✓ Branch 1 taken 1764 times.
 3528 for (u_int node = 0; node < a->body.app_u_u.nodes.node_list_len; node++) {
3500
2/4
✓ Branch 0 taken 1764 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1764 times.
✗ Branch 3 not taken.
 1764 G_INFO("Adding new node to the configuration: %s",
3501 a->body.app_u_u.nodes.node_list_val[node].address)
3502 }
3503
3504 1764 site_def const *old_site = get_site_def();
3505 1764 site_def *site = clone_site_def(old_site);
3506 IFDBG(D_NONE, FN; COPY_AND_FREE_GOUT(dbg_list(&a->body.app_u_u.nodes)););
3507 IFDBG(D_NONE, FN; COPY_AND_FREE_GOUT(dbg_list(&a->body.app_u_u.nodes)););
3508 ADD_DBG(D_BASE, add_event(EVENT_DUMP_PAD, string_arg("a->app_key"));
3509 add_synode_event(a->app_key););
3510
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1764 times.
 1764 assert(old_site);
3511
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1764 times.
 1764 assert(site);
3512 1764 add_site_def(a->body.app_u_u.nodes.node_list_len,
3513 a->body.app_u_u.nodes.node_list_val, site);
3514 1764 set_start_and_boot(site, a);
3515
1/2
✓ Branch 0 taken 1764 times.
✗ Branch 1 not taken.
 1764 if (site->x_proto >= single_writer_support) {
3516 1764 recompute_node_sets(old_site, site);
3517 1764 recompute_timestamps(old_site->detected, &old_site->nodes, site->detected,
3518 1764 &site->nodes);
3519 }
3520 1764 site_install_action(site, a->body.c_t);
3521 1764 return site;
3522 }
3523 }
3524
3525 /**
3526 * Check if we can reconfigure the event horizon.
3527 *
3528 * We can reconfigure the event horizon if all group members support
3529 * reconfiguring the event horizon, and the new event horizon in the domain
3530 * [EVENT_HORIZON_MIN, EVENT_HORIZON_MAX].
3531 *
3532 * We use the group's latest common XCom protocol as a proxy to decide if all
3533 * members support reconfiguring the event horizon.
3534 *
3535 * If the common protocol is at least version 5 (x_1_4) then all members run
3536 * compatible server instances.
3537 *
3538 * Otherwise there are older instances, and it follows that the event horizon
3539 * must be the default and cannot be reconfigured.
3540 */
3541 enum allow_event_horizon_result {
3542 EVENT_HORIZON_ALLOWED,
3543 EVENT_HORIZON_INVALID,
3544 EVENT_HORIZON_UNCHANGEABLE
3545 };
3546 typedef enum allow_event_horizon_result allow_event_horizon_result;
3547
3548 ✗ static void log_event_horizon_reconfiguration_failure(
3549 allow_event_horizon_result error_code,
3550 xcom_event_horizon attempted_event_horizon) {
3551 ✗ switch (error_code) {
3552 ✗ case EVENT_HORIZON_INVALID:
3553 ✗ G_WARNING("The event horizon was not reconfigured to %" PRIu32
3554 "because its domain is [%" PRIu32 ", %" PRIu32 "]",
3555 attempted_event_horizon, xcom_get_minimum_event_horizon(),
3556 xcom_get_maximum_event_horizon());
3557 ✗ break;
3558 ✗ case EVENT_HORIZON_UNCHANGEABLE:
3559 ✗ G_WARNING("The event horizon was not reconfigured to %" PRIu32
3560 " because some of the group's members do not support "
3561 "reconfiguring the event horizon",
3562 attempted_event_horizon);
3563 ✗ break;
3564 ✗ case EVENT_HORIZON_ALLOWED:
3565 ✗ break;
3566 }
3567 }
3568
3569 28 static allow_event_horizon_result allow_event_horizon(
3570 xcom_event_horizon event_horizon) {
3571
2/4
✓ Branch 0 taken 28 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 28 times.
 28 if (event_horizon < EVENT_HORIZON_MIN || event_horizon > EVENT_HORIZON_MAX)
3572 ✗ return EVENT_HORIZON_INVALID;
3573
3574 {
3575 28 const site_def *latest_config = get_site_def();
3576
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 28 times.
 28 if (!reconfigurable_event_horizon(latest_config->x_proto)) {
3577 ✗ assert(backwards_compatible(latest_config->event_horizon));
3578 ✗ return EVENT_HORIZON_UNCHANGEABLE;
3579 }
3580 }
3581 28 return EVENT_HORIZON_ALLOWED;
3582 }
3583
3584 28 static bool_t is_unsafe_event_horizon_reconfiguration(app_data_ptr a) {
3585
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 28 times.
 28 assert(a->body.c_t == set_event_horizon_type);
3586 {
3587 28 xcom_event_horizon new_event_horizon = a->body.app_u_u.event_horizon;
3588 28 bool_t result = FALSE;
3589 allow_event_horizon_result error_code;
3590 28 error_code = allow_event_horizon(new_event_horizon);
3591
1/3
✗ Branch 0 not taken.
✓ Branch 1 taken 28 times.
✗ Branch 2 not taken.
 28 switch (error_code) {
3592 ✗ case EVENT_HORIZON_INVALID:
3593 case EVENT_HORIZON_UNCHANGEABLE:
3594 ✗ log_event_horizon_reconfiguration_failure(error_code,
3595 new_event_horizon);
3596 ✗ result = TRUE;
3597 ✗ break;
3598 28 case EVENT_HORIZON_ALLOWED:
3599 28 break;
3600 }
3601 28 return result;
3602 }
3603 }
3604
3605 // Predicate that checks IF the reconfiguration will be unsafe
3606 156 static bool_t is_unsafe_max_leaders_reconfiguration(app_data_ptr a) {
3607
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 156 times.
 156 assert(a->body.c_t == set_max_leaders);
3608 156 const site_def *latest_config = get_site_def();
3609 156 node_no new_max_leaders = a->body.app_u_u.max_leaders;
3610
2/2
✓ Branch 0 taken 6 times.
✓ Branch 1 taken 150 times.
 156 if (new_max_leaders > get_maxnodes(latest_config)) {
3611
3/6
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 6 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 6 times.
✗ Branch 5 not taken.
 6 G_WARNING("The max number of leaders was not reconfigured to %" PRIu32
3612 " because its domain is [%" PRIu32 ", %" PRIu32 "]",
3613 new_max_leaders, 0, get_maxnodes(latest_config));
3614 6 return TRUE;
3615
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 148 times.
 150 } else if (incompatible_proto_and_max_leaders(latest_config->x_proto,
3616 new_max_leaders)) {
3617
2/4
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 G_WARNING(
3618 "The max number of leaders was not reconfigured "
3619 " because some of the group's members do not support "
3620 "reconfiguring the max number of leaders to %" PRIu32,
3621 new_max_leaders);
3622 2 return TRUE;
3623 } else {
3624 148 return FALSE;
3625 }
3626 }
3627
3628 160 static bool_t is_unsafe_set_leaders_reconfiguration(app_data_ptr a
3629 [[maybe_unused]]) {
3630
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 160 times.
 160 assert(a->body.c_t == set_leaders_type);
3631 160 const site_def *latest_config = get_site_def();
3632
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 151 times.
 160 if (incompatible_proto_and_leaders(latest_config->x_proto)) {
3633
2/4
✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
 9 G_WARNING(
3634 "The set of leaders was not reconfigured "
3635 " because some of the group's members do not support "
3636 "reconfiguring leaders");
3637 9 return TRUE;
3638 } else {
3639 151 return FALSE;
3640 }
3641 }
3642
3643 155 static bool_t is_unsafe_leaders_reconfiguration(app_data_ptr a) {
3644
2/2
✓ Branch 0 taken 304 times.
✓ Branch 1 taken 145 times.
 449 while (a) {
3645
2/3
✓ Branch 0 taken 149 times.
✓ Branch 1 taken 155 times.
✗ Branch 2 not taken.
 304 switch (a->body.c_t) {
3646 149 case set_max_leaders:
3647
2/2
✓ Branch 0 taken 4 times.
✓ Branch 1 taken 145 times.
 149 if (is_unsafe_max_leaders_reconfiguration(a)) return TRUE;
3648 145 break;
3649 155 case set_leaders_type:
3650
2/2
✓ Branch 0 taken 6 times.
✓ Branch 1 taken 149 times.
 155 if (is_unsafe_set_leaders_reconfiguration(a)) return TRUE;
3651 149 break;
3652 ✗ default:
3653 ✗ break;
3654 }
3655 294 a = a->next;
3656 }
3657 145 return FALSE;
3658 }
3659
3660 28 static bool_t are_there_dead_nodes_in_new_config(app_data_ptr a) {
3661
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 28 times.
 28 assert(a->body.c_t == force_config_type);
3662
3663 {
3664 28 u_int nr_nodes_to_add = a->body.app_u_u.nodes.node_list_len;
3665 28 node_address *nodes_to_change = a->body.app_u_u.nodes.node_list_val;
3666 uint32_t i;
3667
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 28 times.
 28 G_DEBUG("Checking for dead nodes in Forced Configuration")
3668
2/2
✓ Branch 0 taken 31 times.
✓ Branch 1 taken 27 times.
 58 for (i = 0; i < nr_nodes_to_add; i++) {
3669 31 node_no node = find_nodeno(get_site_def(), nodes_to_change[i].address);
3670
3671
2/2
✓ Branch 0 taken 27 times.
✓ Branch 1 taken 4 times.
 31 if (node == get_nodeno(get_site_def()))
3672 27 continue; /* No need to validate myself */
3673
3674
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
 4 if (node == VOID_NODE_NO) {
3675 ✗ G_ERROR(
3676 "%s is not in the current configuration."
3677 "Only members in the current configuration can be present"
3678 " in a forced configuration list",
3679 nodes_to_change[i].address)
3680 ✗ return TRUE;
3681 }
3682
3683
2/2
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 3 times.
 4 if (may_be_dead(get_site_def()->detected, node, task_now())) {
3684
2/4
✓ Branch 0 taken 1 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 times.
✗ Branch 3 not taken.
 1 G_ERROR(
3685 "%s is suspected to be failed."
3686 "Only alive members in the current configuration should be "
3687 "present"
3688 " in a forced configuration list",
3689 nodes_to_change[i].address)
3690 1 return TRUE;
3691 }
3692 }
3693 }
3694
3695 27 return FALSE;
3696 }
3697
3698 /**
3699 * Reconfigure the event horizon.
3700 *
3701 * It is possible that concurrent reconfigurations take effect between the
3702 * time this reconfiguration was proposed and now.
3703 *
3704 * Particularly, it is possible that any of the concurrent reconfigurations
3705 * added a new member which does not support reconfiguring the event
3706 * horizon.
3707 *
3708 * We account for these situations by validating if the event horizon
3709 * reconfiguration is still possible under the current state.
3710 *
3711 * If it is not, this reconfiguration does not produce any effect, i.e. no
3712 * new configuration is installed.
3713 */
3714 19 bool_t handle_event_horizon(app_data_ptr a) {
3715
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 19 times.
 19 if (is_unsafe_event_horizon_reconfiguration(a)) return FALSE;
3716
3717 {
3718 19 xcom_event_horizon new_event_horizon = a->body.app_u_u.event_horizon;
3719 19 const site_def *latest_config = get_site_def();
3720 19 site_def *new_config = clone_site_def(latest_config);
3721 IFDBG(D_NONE, FN; NDBG(new_event_horizon, u));
3722 IFDBG(D_NONE, FN; NDBG(new_event_horizon, u));
3723 ADD_DBG(D_BASE, add_event(EVENT_DUMP_PAD, string_arg("a->app_key"));
3724 add_synode_event(a->app_key););
3725
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 19 times.
 19 assert(get_site_def());
3726
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 19 times.
 19 assert(new_config);
3727 19 new_config->event_horizon = new_event_horizon;
3728 19 set_start_and_boot(new_config, a);
3729 19 site_install_action(new_config, a->body.c_t);
3730
2/4
✓ Branch 0 taken 19 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 19 times.
✗ Branch 3 not taken.
 19 G_INFO("The event horizon was reconfigured to %" PRIu32, new_event_horizon);
3731 }
3732 19 return TRUE;
3733 }
3734
3735 109 static bool_t handle_max_leaders(site_def *new_config, app_data_ptr a) {
3736 IFDBG(D_BASE, FN; NUMEXP(a->body.app_u_u.max_leaders));
3737
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 109 times.
 109 assert(new_config);
3738 109 new_config->max_active_leaders = a->body.app_u_u.max_leaders;
3739 109 set_start_and_boot(new_config, a);
3740
2/4
✓ Branch 0 taken 109 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 109 times.
✗ Branch 3 not taken.
 109 G_INFO("Maximum number of leaders was reconfigured to %" PRIu32,
3741 a->body.app_u_u.max_leaders);
3742 109 return TRUE;
3743 }
3744
3745 7 bool_t handle_max_leaders(app_data_ptr a) {
3746
2/2
✓ Branch 0 taken 4 times.
✓ Branch 1 taken 3 times.
 7 if (is_unsafe_max_leaders_reconfiguration(a)) return FALSE;
3747
3748 3 site_def *new_config = clone_site_def(get_site_def());
3749 3 handle_max_leaders(new_config, a);
3750 3 site_install_action(new_config, a->body.c_t);
3751 3 return TRUE;
3752 }
3753
3754 108 static void zero_leader_array(leader_array *l) {
3755 108 l->leader_array_len = 0;
3756 108 l->leader_array_val = nullptr;
3757 108 }
3758
3759 108 static void move_leader_array(leader_array *target, leader_array *source) {
3760 /* Deallocate leader_array from target */
3761 108 xdr_free((xdrproc_t)xdr_leader_array, (char *)target);
3762 108 *target = *source;
3763 /* Zero the source */
3764 108 zero_leader_array(source);
3765 108 }
3766
3767 108 static bool_t handle_set_leaders(site_def *new_config, app_data_ptr a) {
3768 IFDBG(D_BASE, FN; NUMEXP(a->body.app_u_u.leaders.leader_array_len);
3769 NUMEXP(new_config->max_active_leaders));
3770
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 108 times.
 108 assert(new_config);
3771 /* Steal the leaders from a */
3772 108 move_leader_array(&new_config->leaders, &a->body.app_u_u.leaders);
3773 108 set_start_and_boot(new_config, a);
3774 108 return TRUE;
3775 }
3776
3777 5 bool_t handle_set_leaders(app_data_ptr a) {
3778
2/2
✓ Branch 0 taken 3 times.
✓ Branch 1 taken 2 times.
 5 if (is_unsafe_set_leaders_reconfiguration(a)) return FALSE;
3779
3780 2 site_def *new_config = clone_site_def(get_site_def());
3781 2 handle_set_leaders(new_config, a);
3782 2 site_install_action(new_config, a->body.c_t);
3783
3/6
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2 times.
✗ Branch 5 not taken.
 2 G_INFO("Preferred leaders were reconfigured to leaders[0]=%s",
3784 new_config->leaders.leader_array_len > 0
3785 ? new_config->leaders.leader_array_val[0].address
3786 : "n/a");
3787 2 return TRUE;
3788 }
3789
3790 116 bool_t handle_leaders(app_data_ptr a) {
3791
2/2
✓ Branch 0 taken 10 times.
✓ Branch 1 taken 106 times.
 116 if (is_unsafe_leaders_reconfiguration(a)) return FALSE;
3792 106 site_def *new_config = clone_site_def(get_site_def());
3793 106 cargo_type operation{a->body.c_t}; // Deallocate on scope exit if failure
3794 106 bool_t retval = TRUE;
3795
3/4
✓ Branch 0 taken 212 times.
✓ Branch 1 taken 106 times.
✓ Branch 2 taken 212 times.
✗ Branch 3 not taken.
 318 while (a && retval) {
3796
2/3
✓ Branch 0 taken 106 times.
✓ Branch 1 taken 106 times.
✗ Branch 2 not taken.
 212 switch (a->body.c_t) {
3797 106 case set_max_leaders:
3798
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 106 times.
 106 if (!handle_max_leaders(new_config, a)) retval = FALSE;
3799 106 break;
3800 106 case set_leaders_type:
3801
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 106 times.
 106 if (!handle_set_leaders(new_config, a)) retval = FALSE;
3802 106 break;
3803 ✗ default:
3804 ✗ break;
3805 }
3806 212 a = a->next;
3807 }
3808
1/2
✓ Branch 0 taken 106 times.
✗ Branch 1 not taken.
 106 if (retval) {
3809 106 site_install_action(new_config, operation);
3810 } else {
3811 ✗ free_site_def(new_config);
3812 }
3813 106 return retval;
3814 }
3815
3816 2378 void terminate_and_exit() {
3817 IFDBG(D_NONE, FN;);
3818 ADD_DBG(D_FSM, add_event(EVENT_DUMP_PAD, string_arg("terminating"));)
3819
4/4
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 2377 times.
✓ Branch 2 taken 1 times.
✓ Branch 3 taken 2377 times.
 2378 XCOM_FSM(x_fsm_terminate, int_arg(0)); /* Tell xcom to stop */
3820
4/4
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 2377 times.
✓ Branch 2 taken 1 times.
✓ Branch 3 taken 2377 times.
 2378 XCOM_FSM(x_fsm_exit, int_arg(0)); /* Tell xcom to exit */
3821
1/2
✓ Branch 0 taken 2378 times.
✗ Branch 1 not taken.
 2378 if (xcom_expel_cb) xcom_expel_cb(0);
3822 2378 }
3823
3824 2187 static inline int is_empty_site(site_def const *s) {
3825 2187 return s->nodes.node_list_len == 0;
3826 }
3827
3828 3989 site_def *handle_remove_node(app_data_ptr a) {
3829 3989 site_def const *old_site = get_site_def();
3830 3989 site_def *site = clone_site_def(old_site);
3831 IFDBG(D_NONE, FN; COPY_AND_FREE_GOUT(dbg_list(&a->body.app_u_u.nodes)));
3832 ADD_DBG(D_BASE, add_event(EVENT_DUMP_PAD, string_arg("a->app_key"));
3833 add_synode_event(a->app_key);
3834 add_event(EVENT_DUMP_PAD, string_arg("nodeno"));
3835 add_event(EVENT_DUMP_PAD, uint_arg(get_nodeno(site))););
3836
3837 3989 remove_site_def(a->body.app_u_u.nodes.node_list_len,
3838 a->body.app_u_u.nodes.node_list_val, site);
3839 3989 set_start_and_boot(site, a);
3840
1/2
✓ Branch 0 taken 3989 times.
✗ Branch 1 not taken.
 3989 if (site->x_proto >= single_writer_support) {
3841 3989 recompute_node_sets(old_site, site);
3842 3989 recompute_timestamps(old_site->detected, &old_site->nodes, site->detected,
3843 3989 &site->nodes);
3844 }
3845 3989 site_install_action(site, a->body.c_t);
3846 3989 return site;
3847 }
3848
3849 ✗ static void log_ignored_forced_config(app_data_ptr a,
3850 char const *const caller_name) {
3851 ✗ switch (a->body.c_t) {
3852 ✗ case unified_boot_type:
3853 ✗ G_DEBUG("%s: Ignoring a forced intermediate, pending unified_boot",
3854 caller_name);
3855 ✗ break;
3856 ✗ case add_node_type:
3857 ✗ G_DEBUG("%s: Ignoring a forced intermediate, pending add_node for %s",
3858 caller_name, a->body.app_u_u.nodes.node_list_val[0].address);
3859 ✗ break;
3860 ✗ case remove_node_type:
3861 ✗ G_DEBUG("%s: Ignoring a forced intermediate, pending remove_node for %s",
3862 caller_name, a->body.app_u_u.nodes.node_list_val[0].address);
3863 ✗ break;
3864 ✗ case set_event_horizon_type:
3865 ✗ G_DEBUG(
3866 "%s: Ignoring a forced intermediate, pending set_event_horizon for "
3867 "%" PRIu32,
3868 caller_name, a->body.app_u_u.event_horizon);
3869 ✗ break;
3870 ✗ case force_config_type:
3871 ✗ G_DEBUG("%s: Ignoring a forced intermediate, pending force_config",
3872 caller_name);
3873 ✗ break;
3874 ✗ case set_max_leaders:
3875 ✗ G_DEBUG(
3876 "%s: Ignoring a forced intermediate, pending set_max_leaders for "
3877 "%" PRIu32,
3878 caller_name, a->body.app_u_u.max_leaders);
3879 ✗ break;
3880 ✗ case set_leaders_type:
3881 ✗ G_DEBUG("%s: Ignoring a forced intermediate, pending set_leaders_type",
3882 caller_name);
3883 ✗ break;
3884 ✗ case abort_trans:
3885 case app_type:
3886 case begin_trans:
3887 case convert_into_local_server_type:
3888 case disable_arbitrator:
3889 case enable_arbitrator:
3890 case exit_type:
3891 case get_event_horizon_type:
3892 case get_synode_app_data_type:
3893 case prepared_trans:
3894 case remove_reset_type:
3895 case reset_type:
3896 case set_cache_limit:
3897 case view_msg:
3898 case x_terminate_and_exit:
3899 case xcom_boot_type:
3900 case xcom_set_group:
3901 case get_leaders_type:
3902 // Meaningless for any other `cargo_type`s. Ignore.
3903 ✗ break;
3904 }
3905 }
3906
3907 6724 bool_t handle_config(app_data_ptr a, bool const forced) {
3908
6/8
✓ Branch 0 taken 5876 times.
✓ Branch 1 taken 848 times.
✓ Branch 2 taken 5876 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 5772 times.
✓ Branch 5 taken 104 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 5772 times.
 6724 assert(a->body.c_t == unified_boot_type || a->body.c_t == set_max_leaders ||
3909 a->body.c_t == set_leaders_type ||
3910 a->next == nullptr); /* Reconfiguration commands are not batched. */
3911 {
3912 6724 bool_t success = FALSE;
3913
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 6724 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 6724 times.
 6724 if (forced &&
3914 ✗ should_ignore_forced_config_or_view(get_executor_site()->x_proto)) {
3915 ✗ log_ignored_forced_config(a, "handle_config");
3916 ✗ goto end;
3917 }
3918
5/7
✓ Branch 0 taken 848 times.
✓ Branch 1 taken 1764 times.
✓ Branch 2 taken 3989 times.
✓ Branch 3 taken 19 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 104 times.
✗ Branch 6 not taken.
 6724 switch (a->body.c_t) {
3919 848 case unified_boot_type:
3920 848 success = (install_node_group(a) != nullptr);
3921
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 848 times.
 848 assert(success);
3922 848 break;
3923 1764 case add_node_type:
3924 /*
3925 * May fail if meanwhile the event horizon was reconfigured and the
3926 * node is incompatible.
3927 */
3928 1764 success = (handle_add_node(a) != nullptr);
3929 1764 break;
3930 3989 case remove_node_type:
3931 ADD_DBG(D_BASE,
3932 add_event(EVENT_DUMP_PAD, string_arg("got remove_node_type"));)
3933 3989 success = (handle_remove_node(a) != nullptr);
3934
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3989 times.
 3989 assert(success);
3935 3989 break;
3936 19 case set_event_horizon_type:
3937 /* May fail if meanwhile an incompatible node joined. */
3938 19 success = handle_event_horizon(a);
3939 19 break;
3940 ✗ case force_config_type:
3941 ✗ success = (install_node_group(a) != nullptr);
3942 ✗ assert(success);
3943 ✗ break;
3944 104 case set_max_leaders:
3945 case set_leaders_type:
3946 104 success = handle_leaders(a);
3947
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 104 times.
 104 assert(success);
3948 104 break;
3949 ✗ default:
3950 ✗ assert(FALSE); /* Boy oh boy, something is really wrong... */
3951 break;
3952 }
3953 6724 end:
3954 6724 return success;
3955 }
3956 }
3957
3958 7572 static inline int is_member(site_def const *site) {
3959 7572 return site->nodeno != VOID_NODE_NO;
3960 }
3961
3962 /*
3963 Execute xcom message stream.
3964
3965 Beware of the exit logic in this task, which is both simple and
3966 not so simple. Consider three configs C1 and C2. C1 has two
3967 nodes, A and B. C2 has only node B. C3 is empty. A config with
3968 message number N will be activated after a delay of (at least)
3969 alpha messages, where alpha is the size of the pipeline (or the
3970 event horizon).
3971
3972 So, C1.start = C1+alpha, and C2.start = C2+alpha. A, which is re‐
3973 moved from C1, cannot exit until a majority of nodes in the new
3974 config C2 (in this case B) has learned all the messages from con‐
3975 fig C1, which means all messages less than C2.start. How can A
3976 know that a majority of C2 has learned those messages?
3977
3978 If we denote the first message that is not yet decided (and exe‐
3979 cuted) by E, the proposers will not try to propose messages with
3980 number >= E+alpha, and all incoming tcp messages with message
3981 number >= E+alpha will be ignored. E is incremented by the ex‐
3982 ecutor task, so all messages < E are known. This means that when
3983 the value of E+alpha is known, all messages up to and including E
3984 are also known, although not all messages E+1..E+alpha‐1 neces‐
3985 sarily are known.
3986
3987 This leads to the requirement that a node which is removed (A)
3988 needs to wait until it knows the value of C2.start+alpha, since
3989 by then it knows that a majority of the nodes in C2 are ready to
3990 execute C2.start, which in turn implies that a majority of nodes
3991 in C2 knows all the values from config C1. Note that the last
3992 message that should be delivered to the application by a node
3993 that is leaving C1 is C2.start‐1, which is the last message of
3994 C1.
3995
3996 How does a node that is removed get to know values from the next
3997 config? There are two ways, and we use both. First, the node
3998 that tries to exit can simply ask for the message. get_xcom_mes‐
3999 sage() will do this for all messages <= max_synode, but it may
4000 take some time. Second, the nodes of C2 can send the messages
4001 C2.start..C2.start+alpha to the nodes that are removed (nodes
4002 that are in C1 but not in C2). inform_removed() does this. We
4003 take care to handle the case where configs are close enough that
4004 C0 < C1 <= C0+alpha by tracking the oldest config that contains
4005 nodes that are leaving.
4006
4007 This takes care of nodes leaving C1. What about nodes that leave
4008 C2? C3 is empty, so B, which is leaving C2, cannot wait for mes‐
4009 sages from C3. But since C3 is empty, there is no need to wait.
4010 It can exit immediately after having executed C3.start‐1, the
4011 last message of C2. What if C3.start‐1 < C2.start+alpha? This can
4012 happen if C2 and C3 are close. In that case, B will exit before A
4013 gets the chance to learn C2.start+alpha, which will leave A hang‐
4014 ing forever. Clearly, we need to impose an additional constraint,
4015 that C3.start must be greater than C2.start+alpha. This is taken
4016 care of by the special test for an empty config.
4017
4018 Complicated and confusing? Not really, but there is a clean and
4019 simple solution which has not been implemented yet, since it re‐
4020 quires more changes to the consensus logic. If we require that
4021 for the messages C2..C2.start‐1 we have a majority from both the
4022 nodes in C1 and the nodes in C2, the nodes not in C2 can exit
4023 when they have executed message C2.start‐1, since we then know
4024 that a majority of the nodes of C2 has agreed on those messages
4025 as well, so they do not depend on the nodes not in C2 any more.
4026 This holds even if C2 is empty. Note that requiring a majority
4027 from both C1 and C2 is different from requiring a majority from
4028 C1+C2, which means that the proposer logic needs to consider an‐
4029 swers from two different sets of acceptors for those messages.
4030 Since acceptors are identified by their node number, and the node
4031 numbers need not be the same for both configs, we need to main‐
4032 tain a mapping between the nodes numbers of any two consecutive
4033 configs. Alternatively, we could remove the node numbers alto‐
4034 gether, and always use a unique, unchanging ID for a node, like
4035 IP address + port.
4036
4037 TODO:
4038
4039 Move the delayed delivery logic into MCM-specific code, since it is
4040 only needed by MCM. Is it still needed?
4041
4042 Rewrite exit logic as FSM with more states. (RUN, EMPTY_EXIT,
4043 NOT_MEMBER_EXIT) to avoid unnecessary tests.
4044
4045 */
4046
4047 /* FIFO which tracks the message numbers where we should deliver queued messages
4048 or
4049 inform the removed nodes */
4050 #define FIFO_SIZE 1000
4051 static struct {
4052 int n;
4053 int front;
4054 int rear;
4055 synode_no q[FIFO_SIZE];
4056 } delay_fifo;
4057
4058 13448 static inline int addone(int i) { return ((i + 1) % FIFO_SIZE); }
4059
4060 /* Is queue empty? */
4061 796686 static inline int fifo_empty() { return delay_fifo.n <= 0; }
4062
4063 /* Is queue full? */
4064 6724 static inline int fifo_full() { return delay_fifo.n >= FIFO_SIZE; }
4065
4066 /* Insert in queue */
4067 6724 static inline void fifo_insert(synode_no s) {
4068
1/2
✓ Branch 0 taken 6724 times.
✗ Branch 1 not taken.
 6724 if (!fifo_full()) {
4069 6724 delay_fifo.n++;
4070 6724 delay_fifo.q[delay_fifo.rear] = s;
4071 6724 delay_fifo.rear = addone(delay_fifo.rear);
4072 }
4073 6724 }
4074
4075 /* Extract first from queue */
4076 6724 static inline synode_no fifo_extract() {
4077
1/2
✓ Branch 0 taken 6724 times.
✗ Branch 1 not taken.
 6724 if (!fifo_empty()) {
4078 6724 synode_no ret = delay_fifo.q[delay_fifo.front];
4079 6724 delay_fifo.front = addone(delay_fifo.front);
4080 6724 delay_fifo.n--;
4081 6724 return ret;
4082 } else {
4083 ✗ return null_synode;
4084 }
4085 }
4086
4087 /* Return first in queue, but do not dequeue */
4088 246130 static inline synode_no fifo_front() {
4089
1/2
✓ Branch 0 taken 246130 times.
✗ Branch 1 not taken.
 246130 if (!fifo_empty()) {
4090 246130 return delay_fifo.q[delay_fifo.front];
4091 } else {
4092 ✗ return null_synode;
4093 }
4094 }
4095
4096 struct execute_context;
4097 typedef struct execute_context execute_context;
4098
4099 typedef void (*exec_fp)(execute_context *xc);
4100
4101 struct execute_context {
4102 pax_machine *p;
4103 int n;
4104 int old_n;
4105 double old_t;
4106 synode_no exit_synode;
4107 synode_no delivery_limit;
4108 exec_fp state;
4109 int exit_flag; /* To avoid state explosion */
4110 int inform_index;
4111 };
4112
4113 static void dump_exec_state(execute_context *xc [[maybe_unused]],
4114 long dbg [[maybe_unused]]);
4115 static int x_check_exit(execute_context *xc);
4116 static int x_check_execute_inform(execute_context *xc);
4117 static void x_fetch(execute_context *xc);
4118 static void x_execute(execute_context *xc);
4119 static void x_check_increment_fetch(execute_context *xc);
4120 static void x_check_increment_execute(execute_context *xc);
4121 static void x_terminate(execute_context *xc);
4122
4123 struct fp_name {
4124 exec_fp fp;
4125 char const *name;
4126 };
4127
4128 #define NAME(f) \
4129 { f, #f }
4130
4131 /* List of fp, name pairs */
4132 static struct fp_name MY_ATTRIBUTE((unused)) oblist[] = {
4133 NAME(x_fetch), NAME(x_execute), NAME(x_terminate), {nullptr, nullptr}};
4134 #undef NAME
4135
4136 #if TASK_DBUG_ON
4137 /* purecov: begin deadcode */
4138 char const *get_fp_name(exec_fp fp) {
4139 struct fp_name *list = oblist;
4140 while (list->fp) {
4141 if (list->fp == fp) return list->name;
4142 list++;
4143 }
4144 return "no such fp";
4145 }
4146 /* purecov: end */
4147 #endif
4148
4149 6724 static void setup_exit_handling(execute_context *xc, site_def *site) {
4150 synode_no delay_until;
4151
2/2
✓ Branch 0 taken 4537 times.
✓ Branch 1 taken 2187 times.
 6724 if (is_member(site)) {
4152 4537 delay_until = compute_delay(site->start, site->event_horizon);
4153 } else { /* Not in this site */
4154 /* See if site will be empty when we leave. If the new site
4155 * is empty, we should exit after having delivered the last
4156 * message from the old site. */
4157
4158 /* Note limit of delivery. We should never deliver anything after the
4159 * start of the next site. */
4160 2187 xc->delivery_limit = site->start;
4161
4162 /* If we are not a member of the new site, we should exit
4163 after having seen enough messages beyond the end of the current site.
4164 This ensures that a majority of the next site will have agreed upon all
4165 messages that belong to the current site.
4166 */
4167 2187 xc->exit_synode = compute_delay(site->start, site->event_horizon);
4168
2/2
✓ Branch 0 taken 888 times.
✓ Branch 1 taken 1299 times.
 2187 if (is_empty_site(site)) {
4169 /* If site is empty, increase start to allow nodes to terminate before
4170 * start. This works as if there was a non-empty group after the
4171 * exit_synode, effectively allowing the majority of the current group
4172 * to agree on all messages up to exit_synode.
4173 */
4174 888 site->start = compute_delay(
4175 compute_delay(site->start, site->event_horizon), site->event_horizon);
4176 }
4177
3/4
✓ Branch 0 taken 2187 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2157 times.
✓ Branch 3 taken 30 times.
 2187 if (!synode_lt(xc->exit_synode, max_synode)) {
4178 /* We need messages from the next site, so set max_synode accordingly.
4179 */
4180
2/4
✓ Branch 0 taken 2157 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2157 times.
✗ Branch 3 not taken.
 2157 set_max_synode(incr_synode(xc->exit_synode));
4181 }
4182 /* Note where we switch to execute and inform removed nodes */
4183 2187 delay_until = xc->exit_synode;
4184
4185 IFDBG(D_EXEC, FN; SYCEXP(delay_until); SYCEXP(executed_msg);
4186 SYCEXP(max_synode));
4187 IFDBG(D_EXEC, FN; SYCEXP(xc->exit_synode); SYCEXP(executed_msg);
4188 SYCEXP(max_synode));
4189
4190 /* Note that we will exit */
4191 2187 xc->exit_flag = 1;
4192 }
4193
4194 /* Ensure that max_synode is greater than trigger for delivery
4195 */
4196
3/4
✓ Branch 0 taken 6724 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 4511 times.
✓ Branch 3 taken 2213 times.
 6724 if (synode_gt(delay_until, max_synode))
4197
2/4
✓ Branch 0 taken 4511 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 4511 times.
✗ Branch 3 not taken.
 4511 set_max_synode(incr_msgno(delay_until));
4198 6724 fifo_insert(delay_until);
4199 6724 (xc->inform_index)++;
4200
4201 /* If I am the leader, will propose no-ops until current max_synode
4202 */
4203 6724 }
4204
4205 /* Called immediately after we have got a new message.
4206 Terminate if we have no site.
4207 Otherwise, handle config messages immediately.
4208 Afterwards, switch to check_exit_fetch. */
4209 527269 static void x_fetch(execute_context *xc) {
4210 /* Execute unified_boot immediately, but do not deliver site message
4211 * until we are ready to execute messages from the new site
4212 * definition. At that point we can be certain that a majority have
4213 * learned everything from the old site. */
4214
4215 527269 app_data *app = xc->p->learner.msg->a;
4216
6/6
✓ Branch 0 taken 159433 times.
✓ Branch 1 taken 367836 times.
✓ Branch 2 taken 6754 times.
✓ Branch 3 taken 152679 times.
✓ Branch 4 taken 6724 times.
✓ Branch 5 taken 520545 times.
 534023 if (app && is_config(app->body.c_t) &&
4217
2/2
✓ Branch 0 taken 6724 times.
✓ Branch 1 taken 30 times.
 6754 synode_gt(executed_msg, get_site_def()->boot_key)) /* Redo test */
4218 {
4219 6724 site_def *site = nullptr;
4220 bool_t reconfiguration_successful =
4221 6724 handle_config(app, (xc->p->learner.msg->force_delivery != 0));
4222
1/2
✓ Branch 0 taken 6724 times.
✗ Branch 1 not taken.
 6724 if (reconfiguration_successful) {
4223 /* If the reconfiguration failed then it does not have any
4224 * effect. What follows only makes sense if the reconfiguration
4225 * took effect. */
4226
1/2
✓ Branch 0 taken 6724 times.
✗ Branch 1 not taken.
 6724 set_last_received_config(executed_msg);
4227
1/2
✓ Branch 0 taken 6724 times.
✗ Branch 1 not taken.
 6724 synode_no min_synode = min_proposer_synode();
4228
3/6
✓ Branch 0 taken 6724 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 6724 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 6724 times.
 13448 if (synode_eq(null_synode, min_synode) ||
4229
2/4
✓ Branch 0 taken 6724 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 6724 times.
 6724 synode_lt(delivered_msg, min_synode))
4230 ✗ min_synode = get_last_delivered_msg();
4231
1/2
✓ Branch 0 taken 6724 times.
✗ Branch 1 not taken.
 6724 garbage_collect_site_defs(min_synode);
4232
1/2
✓ Branch 0 taken 6724 times.
✗ Branch 1 not taken.
 6724 site = get_site_def_rw();
4233
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6724 times.
 6724 if (site == nullptr) {
4234 ✗ xc->state = x_terminate;
4235 ✗ return;
4236 }
4237 IFDBG(D_EXEC, FN; STRLIT("new config "); SYCEXP(site->boot_key););
4238
4239
1/2
✓ Branch 0 taken 6724 times.
✗ Branch 1 not taken.
 6724 if (xc->exit_flag == 0) {
4240 /* We have not yet set the exit trigger */
4241
1/2
✓ Branch 0 taken 6724 times.
✗ Branch 1 not taken.
 6724 setup_exit_handling(xc, site);
4242 }
4243 }
4244 } else {
4245 IFDBG(D_EXEC, FN; SYCEXP(executed_msg); SYCEXP(get_site_def()->boot_key));
4246 }
4247 /* Check for exit and increment executed_msg */
4248 527269 x_check_increment_fetch(xc);
4249 }
4250
4251 /* Push messages to nodes that have been removed.
4252 Signal switch to execute when nothing left to push by returning 1 */
4253 530384 static int x_check_execute_inform(execute_context *xc) {
4254 IFDBG(D_EXEC, FN; SYCEXP(fifo_front()); SYCEXP(executed_msg);
4255 SYCEXP(xc->exit_synode); NDBG(xc->exit_flag, d));
4256
2/2
✓ Branch 0 taken 291038 times.
✓ Branch 1 taken 239346 times.
 530384 if (fifo_empty()) {
4257 291038 return 1;
4258
2/2
✓ Branch 0 taken 6724 times.
✓ Branch 1 taken 232622 times.
 239346 } else if (!synode_lt(executed_msg, fifo_front())) {
4259 6724 while (
4260
4/4
✓ Branch 0 taken 6784 times.
✓ Branch 1 taken 6664 times.
✓ Branch 2 taken 6724 times.
✓ Branch 3 taken 6724 times.
 20232 !fifo_empty() &&
4261
2/2
✓ Branch 0 taken 6724 times.
✓ Branch 1 taken 60 times.
 6784 !synode_lt(executed_msg, fifo_front())) { /* More than one may match */
4262 6724 inform_removed(xc->inform_index, 0);
4263 6724 fifo_extract();
4264 6724 (xc->inform_index)--;
4265 }
4266 6724 garbage_collect_servers();
4267 6724 return 1;
4268 }
4269 232622 dump_exec_state(xc, D_EXEC);
4270 232622 return 0;
4271 }
4272
4273 /* Check for exit and return 1 if we should exit. */
4274 1036093 static int x_check_exit(execute_context *xc) {
4275 /* See if we should exit when having seen this message */
4276
4/4
✓ Branch 0 taken 115617 times.
✓ Branch 1 taken 920476 times.
✓ Branch 2 taken 45154 times.
✓ Branch 3 taken 70463 times.
 1081247 return (xc->exit_flag && !synode_lt(executed_msg, xc->exit_synode) &&
4277
2/2
✓ Branch 0 taken 2187 times.
✓ Branch 1 taken 42967 times.
 1081247 !synode_lt(delivered_msg, xc->delivery_limit));
4278 }
4279
4280 /* Terminate if we should exit, else increment executed_msg and see if we
4281 * should switch to execute */
4282 530384 static void x_check_increment_fetch(execute_context *xc) {
4283
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 530384 times.
 530384 if (x_check_exit(xc)) {
4284 ✗ xc->state = x_terminate;
4285 } else {
4286 530384 SET_EXECUTED_MSG(incr_synode(executed_msg));
4287
2/2
✓ Branch 0 taken 297762 times.
✓ Branch 1 taken 232622 times.
 530384 if (x_check_execute_inform(xc)) {
4288 297762 xc->state = x_execute;
4289 }
4290 }
4291 530384 }
4292
4293 /* Terminate if we should exit, else increment delivered_msg and see if we
4294 * should switch to fetch */
4295 505709 static void x_check_increment_execute(execute_context *xc) {
4296
2/2
✓ Branch 0 taken 2187 times.
✓ Branch 1 taken 503522 times.
 505709 if (x_check_exit(xc)) {
4297 2187 xc->state = x_terminate;
4298 } else {
4299 /* Increment delivered_msg and switch to fetch if delivered_msg equals
4300 * executed_msg; */
4301 503522 delivered_msg = incr_synode(delivered_msg);
4302
2/2
✓ Branch 0 taken 295575 times.
✓ Branch 1 taken 207947 times.
 503522 if (synode_eq(delivered_msg, executed_msg)) {
4303 295575 xc->state = x_fetch;
4304 }
4305 }
4306 505709 }
4307
4308 /* Deliver one message if it should be delivered. Switch state to see if
4309 we should exit */
4310 505709 static void x_execute(execute_context *xc) {
4311 505709 site_def *x_site = find_site_def_rw(delivered_msg);
4312
4313 IFDBG(D_EXEC, FN; SYCEXP(delivered_msg); SYCEXP(delivered_msg);
4314 SYCEXP(executed_msg); SYCEXP(xc->exit_synode); NDBG(xc->exit_flag, d));
4315 505709 if (!is_cached(delivered_msg)) {
4316 /* purecov: begin deadcode */
4317 #ifdef TASK_EVENT_TRACE
4318 dump_task_events();
4319 #endif
4320 /* purecov: end */
4321 }
4322
2/2
✓ Branch 0 taken 502614 times.
✓ Branch 1 taken 3095 times.
 505709 if (!ignore_message(delivered_msg, x_site, "x_execute")) {
4323
1/2
✓ Branch 0 taken 502614 times.
✗ Branch 1 not taken.
 502614 assert(is_cached(delivered_msg) && "delivered_msg should have been cached");
4324 502614 xc->p = get_cache(delivered_msg);
4325
2/2
✓ Branch 0 taken 159374 times.
✓ Branch 1 taken 343240 times.
 502614 if ((xc->p)->learner.msg->msg_type != no_op) {
4326 /* Avoid delivery after start if we should exit */
4327
6/6
✓ Branch 0 taken 2256 times.
✓ Branch 1 taken 157118 times.
✓ Branch 2 taken 2243 times.
✓ Branch 3 taken 13 times.
✓ Branch 4 taken 159361 times.
✓ Branch 5 taken 13 times.
 159374 if (xc->exit_flag == 0 || synode_lt(delivered_msg, xc->delivery_limit)) {
4328 IFDBG(D_EXEC, FN; STRLIT("executing "); SYCEXP(delivered_msg);
4329 SYCEXP(executed_msg); SYCEXP(xc->delivery_limit);
4330 NDBG(xc->exit_flag, d));
4331 159361 last_delivered_msg = delivered_msg;
4332 159361 execute_msg(find_site_def_rw(delivered_msg), xc->p, xc->p->learner.msg);
4333 }
4334 }
4335 }
4336 /* Garbage collect old servers */
4337
2/2
✓ Branch 0 taken 7245 times.
✓ Branch 1 taken 498464 times.
 505709 if (synode_eq(delivered_msg, x_site->start)) {
4338 7245 garbage_collect_servers();
4339 }
4340 #if defined(TASK_DBUG_ON) && TASK_DBUG_ON
4341 IFDBG(D_EXEC, perf_dbg(&xc->n, &xc->old_n, &xc->old_t));
4342 #endif
4343 /* Check for exit and increment delivered_msg */
4344 505709 x_check_increment_execute(xc);
4345 505709 }
4346
4347 static execute_context *debug_xc;
4348
4349 1128778 static void dump_exec_state(execute_context *xc [[maybe_unused]],
4350 long dbg [[maybe_unused]]) {
4351 IFDBG(dbg, FN; SYCEXP(executed_msg); SYCEXP(delivered_msg);
4352 SYCEXP(max_synode); SYCEXP(last_delivered_msg); NDBG(delay_fifo.n, d);
4353 NDBG(delay_fifo.front, d); NDBG(delay_fifo.rear, d);
4354 SYCEXP(fifo_front()); SYCEXP(xc->exit_synode);
4355 SYCEXP(xc->delivery_limit); NDBG(xc->exit_flag, d);
4356 NDBG(xc->inform_index, d); NDBG(prop_started, d);
4357 NDBG(prop_finished, d););
4358 1128778 }
4359
4360 889574 static void dump_debug_exec_state() {
4361
1/2
✓ Branch 0 taken 889574 times.
✗ Branch 1 not taken.
 889574 if (debug_xc) dump_exec_state(debug_xc, D_EXEC);
4362 889574 }
4363
4364 /* Terminate the excutor_task. */
4365 2187 static void x_terminate(execute_context *xc) {
4366 2187 dump_exec_state(xc, D_BUG);
4367 2187 xc->state = nullptr;
4368 2187 }
4369
4370 364492 static int executor_task(task_arg arg [[maybe_unused]]) {
4371 DECL_ENV
4372 execute_context xc;
4373 2215 ENV_INIT
4374 2215 END_ENV_INIT
4375 END_ENV;
4376 /* xcom_debug_mask = D_BUG; */
4377 IFDBG(D_EXEC, FN; NDBG(stack->sp->state, d); SYCEXP(executed_msg););
4378
6/10
✓ Branch 0 taken 2215 times.
✓ Branch 1 taken 360090 times.
✓ Branch 2 taken 2187 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 2215 times.
✓ Branch 6 taken 2215 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✓ Branch 9 taken 2215 times.
 364492 TASK_BEGIN
4379 2215 ep->xc.p = nullptr;
4380 2215 ep->xc.n = 0;
4381 2215 ep->xc.old_n = 0;
4382 2215 ep->xc.old_t = task_now();
4383 2215 ep->xc.exit_synode = null_synode;
4384 2215 ep->xc.delivery_limit = null_synode;
4385 2215 ep->xc.exit_flag = 0;
4386 2215 ep->xc.inform_index = -1;
4387 2215 delay_fifo.n = 0;
4388 2215 delay_fifo.front = 0;
4389 2215 delay_fifo.rear = 0;
4390 2215 debug_xc = &ep->xc;
4391
4392
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2215 times.
 2215 if (executed_msg.msgno == 0) executed_msg.msgno = 1;
4393 2215 delivered_msg = executed_msg;
4394 2215 ep->xc.state = x_fetch;
4395 2215 executor_site = find_site_def_rw(executed_msg);
4396
4397 /* The following loop implements a state machine based on function pointers,
4398 effectively acting as non-local gotos.
4399 The functions all operate on data in the execution context xc, and
4400 switch state by setting xc->state to the function corresponding to the
4401 new state.
4402 */
4403
3/4
✓ Branch 0 taken 1040495 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1038308 times.
✓ Branch 3 taken 2187 times.
 1040495 while (!xcom_shutdown && ep->xc.state != nullptr) {
4404 IFDBG(D_EXEC, FN; STRLIT(get_fp_name(ep->xc.state)););
4405
2/2
✓ Branch 0 taken 530412 times.
✓ Branch 1 taken 507896 times.
 1038308 if (ep->xc.state == x_fetch) { /* Special case because of task macros */
4406
2/2
✓ Branch 0 taken 3115 times.
✓ Branch 1 taken 527297 times.
 530412 if (ignore_message(executed_msg, executor_site, "executor_task")) {
4407 IFDBG(D_EXEC, FN; STRLIT("ignoring message "); SYCEXP(executed_msg));
4408 3115 x_check_increment_fetch(&ep->xc); /* Just increment past losers */
4409 } else {
4410 IFDBG(D_EXEC, FN; STRLIT("fetching message "); SYCEXP(executed_msg));
4411
10/14
✓ Branch 0 taken 527290 times.
✓ Branch 1 taken 360097 times.
✓ Branch 2 taken 21 times.
✓ Branch 3 taken 527269 times.
✓ Branch 4 taken 360097 times.
✓ Branch 5 taken 527269 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 360090 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 360090 times.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✓ Branch 12 taken 360090 times.
✓ Branch 13 taken 527269 times.
 1247477 TASK_CALL(get_xcom_message(&ep->xc.p, executed_msg, FIND_MAX));
4412 IFDBG(D_EXEC, FN; STRLIT("got message "); SYCEXP(ep->xc.p->synode);
4413 COPY_AND_FREE_GOUT(dbg_app_data(ep->xc.p->learner.msg->a)));
4414 527269 x_fetch(&ep->xc);
4415 }
4416 } else {
4417 507896 ep->xc.state(&ep->xc);
4418 }
4419 }
4420
4421 /* Inform all removed nodes before we exit */
4422 ADD_DBG(D_FSM, add_event(EVENT_DUMP_PAD, string_arg("terminating"));)
4423 2187 inform_removed(ep->xc.inform_index, 1);
4424 2187 dump_exec_state(&ep->xc, D_EXEC);
4425
4426 #ifndef NO_DELAYED_TERMINATION
4427 IFDBG(D_EXEC, FN; STRLIT("delayed terminate and exit"));
4428
4429 /* Wait to allow messages to propagate */
4430
3/6
✗ Branch 0 not taken.
✓ Branch 1 taken 2187 times.
✓ Branch 2 taken 2187 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 2187 times.
 4374 TASK_DELAY(TERMINATE_DELAY);
4431
4432 /* Start termination of xcom */
4433 2187 terminate_and_exit();
4434 #endif
4435
4436 2208 FINALLY
4437 2208 dump_exec_state(&ep->xc, D_EXEC);
4438 IFDBG(D_BUG, FN; STRLIT(" shutdown "); SYCEXP(executed_msg);
4439 NDBG(task_now(), f));
4440
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2208 times.
 2208 TASK_END;
4441 }
4442
4443 9209 static synode_no get_sweep_start() {
4444 9209 synode_no find = executed_msg;
4445 9209 find.node = get_nodeno(find_site_def(find));
4446
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 9209 times.
 9209 if (find.node < executed_msg.node) {
4447 ✗ find = incr_msgno(find);
4448 }
4449 9209 return find;
4450 }
4451
4452 /* Allow takeover of channel if not all are leaders. We may need to adjust
4453 * this if we allow any subset of the nodes as leaders */
4454 156571 static bool allow_channel_takeover(site_def const *site) {
4455 156571 return site->max_active_leaders != active_leaders_all;
4456 }
4457
4458 156571 static void broadcast_noop(synode_no find, pax_machine *p) {
4459 156571 site_def const *site = find_site_def(find);
4460
4461 /*
4462 If we allow channel hijacking, we cannot send skip_op, but need consensus.
4463 There are two options here:
4464
4465 a) We unconditionally propose a `no_op` using the regular 3-phase Paxos
4466 protocol, or
4467 b) We propose a `no_op` using the 2-phase Paxos protocol *if* we
4468 are sure that no other Proposer will try to run the 2-phase Paxos
4469 protocol on `find`. If we are not sure, we propose using the 3-phase Paxos
4470 protocol.
4471
4472 Option (a) is always safe, but we pay the cost of 3-phase Paxos.
4473 Option (b) can be implemented by having the leaders keep track of the
4474 synods they allocate to non-leaders. If we are the leader for `find` and we
4475 allocated it to a non-leader, we must use 3-phase Paxos here to be safe
4476 against the non-leader using 2-phase Paxos. If we never allocated `find` to
4477 a non-leader, we can use 2-phase Paxos here if we ensure we don't allocate
4478 `find` to a non-leader afterwards.
4479
4480 We go with option (a) because there is no evidence that the additional
4481 complexity that option (b) requires is worthwhile.
4482 */
4483
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 156571 times.
 156571 if (allow_channel_takeover(site)) {
4484 ✗ propose_noop(find, p); // Single leader
4485 } else {
4486 156571 skip_msg(pax_msg_new(find, site)); // Multiple leaders
4487 }
4488 156571 }
4489
4490 723859 static int sweeper_task(task_arg arg [[maybe_unused]]) {
4491 DECL_ENV
4492 synode_no find;
4493 2215 ENV_INIT
4494 2215 END_ENV_INIT
4495 END_ENV;
4496
4497
6/10
✓ Branch 0 taken 2215 times.
✓ Branch 1 taken 324812 times.
✓ Branch 2 taken 396832 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 2215 times.
✓ Branch 6 taken 2215 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✓ Branch 9 taken 2215 times.
 723859 TASK_BEGIN
4498
4499 2215 ep->find = get_sweep_start();
4500
4501
1/2
✓ Branch 0 taken 721651 times.
✗ Branch 1 not taken.
 721651 while (!xcom_shutdown) {
4502 721651 ep->find.group_id =
4503 721651 executed_msg.group_id; /* In case group id has changed */
4504 #ifndef AGGRESSIVE_SWEEP
4505 while (!is_only_task()) {
4506 TASK_YIELD;
4507 }
4508 #endif
4509 ADD_DBG(D_NONE, add_event(EVENT_DUMP_PAD, string_arg("sweeper ready"));
4510 add_synode_event(executed_msg););
4511 /* IFDBG(D_NONE, FN; STRLIT("ready to run "); */
4512 /* SYCEXP(executed_msg); SYCEXP(max_synode);
4513 * SYCEXP(ep->find));
4514 */
4515
6/6
✓ Branch 0 taken 638197 times.
✓ Branch 1 taken 324094 times.
✓ Branch 2 taken 437594 times.
✓ Branch 3 taken 200603 times.
✓ Branch 4 taken 437594 times.
✓ Branch 5 taken 524697 times.
 962291 while (synode_lt(ep->find, max_synode) && !too_far(ep->find)) {
4516 /* pax_machine * pm = hash_get(ep->find); */
4517 437594 pax_machine *pm = nullptr;
4518 ADD_DBG(D_NONE,
4519 add_event(EVENT_DUMP_PAD, string_arg("sweeper examining"));
4520 add_synode_event(ep->find););
4521
2/2
✓ Branch 0 taken 196954 times.
✓ Branch 1 taken 240640 times.
 437594 if (ep->find.node == VOID_NODE_NO) {
4522
2/2
✓ Branch 0 taken 6994 times.
✓ Branch 1 taken 189960 times.
 196954 if (synode_gt(executed_msg, ep->find)) {
4523 6994 ep->find = get_sweep_start();
4524 }
4525
1/2
✓ Branch 0 taken 196954 times.
✗ Branch 1 not taken.
 196954 if (ep->find.node == VOID_NODE_NO) goto deactivate;
4526 }
4527 240640 pm = get_cache(ep->find);
4528 ADD_DBG(D_CONS, add_event(EVENT_DUMP_PAD, string_arg("sweeper checking"));
4529 add_synode_event(ep->find);
4530 add_event(EVENT_DUMP_PAD, string_arg(pax_op_to_str(pm->op)));
4531 add_event(EVENT_DUMP_PAD, string_arg("pm"));
4532 add_event(EVENT_DUMP_PAD, void_arg(pm)););
4533
3/4
✓ Branch 0 taken 240640 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 239982 times.
✓ Branch 3 taken 658 times.
 240640 if (pm && !pm->force_delivery) { /* We want full 3 phase Paxos for
4534 forced messages */
4535 ADD_DBG(
4536 D_CONS, add_event(EVENT_DUMP_PAD, string_arg("sweeper checking"));
4537 add_synode_event(ep->find);
4538 add_event(EVENT_DUMP_PAD, string_arg(pax_op_to_str(pm->op)));
4539 add_event(EVENT_DUMP_PAD, string_arg("is_busy_machine"));
4540 add_event(EVENT_DUMP_PAD, int_arg(is_busy_machine(pm)));
4541 add_event(EVENT_DUMP_PAD, string_arg("pm->acceptor.promise.cnt"));
4542 add_event(EVENT_DUMP_PAD, int_arg(pm->acceptor.promise.cnt));
4543 add_event(EVENT_DUMP_PAD, string_arg("finished(pm)"));
4544 add_event(EVENT_DUMP_PAD, int_arg(finished(pm)));
4545 add_event(EVENT_DUMP_PAD, string_arg("pm->acceptor.msg"));
4546 add_event(EVENT_DUMP_PAD, void_arg(pm->acceptor.msg)););
4547 /* IFDBG(D_NONE, FN; dbg_pax_machine(pm)); */
4548
2/2
✓ Branch 0 taken 235864 times.
✓ Branch 1 taken 1597 times.
 477443 if (!is_busy_machine(pm) && pm->acceptor.promise.cnt == 0 &&
4549
7/8
✓ Branch 0 taken 237461 times.
✓ Branch 1 taken 2521 times.
✓ Branch 2 taken 157500 times.
✓ Branch 3 taken 78364 times.
✓ Branch 4 taken 157500 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 157500 times.
✓ Branch 7 taken 82482 times.
 477443 !pm->acceptor.msg && !finished(pm)) {
4550 ADD_DBG(
4551 D_CONS, add_event(EVENT_DUMP_PAD, string_arg("sweeper skipping"));
4552 add_synode_event(ep->find);
4553 add_event(EVENT_DUMP_PAD, string_arg(pax_op_to_str(pm->op))););
4554 157500 site_def *config = find_site_def_rw(ep->find);
4555 // Do not send noop if single writer, since there normally will be
4556 // no holes in the message sequence, and it may interfere with
4557 // messages delegated to secondary nodes.
4558
4/4
✓ Branch 0 taken 156571 times.
✓ Branch 1 taken 929 times.
✓ Branch 2 taken 156571 times.
✓ Branch 3 taken 929 times.
 314071 if (config->max_active_leaders != 1 &&
4559
1/2
✓ Branch 0 taken 156571 times.
✗ Branch 1 not taken.
 156571 !ignore_message(ep->find, config, "sweeper_task")) {
4560 156571 broadcast_noop(ep->find, pm);
4561 }
4562 IFDBG(D_NONE, FN; STRLIT("skipping "); SYCEXP(ep->find));
4563 }
4564 }
4565 240640 ep->find = incr_msgno(ep->find);
4566 }
4567 524697 deactivate:
4568
2/2
✓ Branch 0 taken 324819 times.
✓ Branch 1 taken 396832 times.
 721651 if (!synode_lt(ep->find, max_synode)) {
4569
4/6
✗ Branch 0 not taken.
✓ Branch 1 taken 324812 times.
✓ Branch 2 taken 324812 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1634 times.
✓ Branch 5 taken 323178 times.
 649631 TASK_DEACTIVATE;
4570 } else {
4571
4/6
✗ Branch 0 not taken.
✓ Branch 1 taken 396832 times.
✓ Branch 2 taken 396832 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 574 times.
✓ Branch 5 taken 396258 times.
 793664 TASK_DELAY(0.010); /* Let poll_wait check for IO */
4572 }
4573 }
4574 ✗ FINALLY
4575 IFDBG(D_BUG, FN; STRLIT(" shutdown sweeper "); SYCEXP(executed_msg);
4576 NDBG(task_now(), f));
4577
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2208 times.
 2208 TASK_END;
4578 }
4579
4580 448806 static double wakeup_delay(double old) {
4581 448806 double const minimum_threshold = 0.1;
4582 #ifdef EXECUTOR_TASK_AGGRESSIVE_NO_OP
4583 double const maximum_threshold = 1.0;
4584 #else
4585 448806 double const maximum_threshold = 20.0;
4586 #endif /* EXECUTOR_TASK_AGGRESSIVE_NO_OP */
4587 448806 double retval = 0.0;
4588
2/2
✓ Branch 0 taken 271876 times.
✓ Branch 1 taken 176930 times.
 448806 if (0.0 == old) {
4589 271876 double m = median_time();
4590 271876 double const fuzz = 5.0;
4591 IFDBG(D_BUG, FN; NDBG(m, f));
4592 // Guard against unreasonable estimates of median consensus time
4593
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 271876 times.
 271876 if (m <= 0.0) m = minimum_threshold;
4594
2/2
✓ Branch 0 taken 643 times.
✓ Branch 1 taken 271233 times.
 271876 if (m > maximum_threshold / fuzz) m = (maximum_threshold / fuzz) / 2.0;
4595 271876 retval = minimum_threshold + fuzz * m + m * xcom_drand48();
4596 } else {
4597 176930 retval = old * 1.4142136; /* Exponential backoff */
4598 }
4599 /* If we exceed maximum, choose a random value in the max/2..max interval */
4600
2/2
✓ Branch 0 taken 223 times.
✓ Branch 1 taken 448583 times.
 448806 if (retval > maximum_threshold) {
4601 223 double const low = maximum_threshold / 2.0;
4602 223 retval = low + xcom_drand48() * (maximum_threshold - low);
4603 }
4604 IFDBG(D_BUG, FN; NDBG(retval, f));
4605 448806 return retval;
4606 }
4607
4608 19228 static site_def const *init_noop(synode_no find, pax_machine *p) {
4609 /* Prepare to send a noop */
4610 19228 site_def const *site = find_site_def(find);
4611 IFDBG(D_NONE, FN; SYCEXP(find); SYCEXP(executed_msg));
4612
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 19228 times.
 19228 assert(!too_far(find));
4613 19228 replace_pax_msg(&p->proposer.msg, pax_msg_new(find, site));
4614
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 19228 times.
 19228 assert(p->proposer.msg);
4615 19228 create_noop(p->proposer.msg);
4616 19228 return site;
4617 }
4618
4619 19228 static void propose_noop(synode_no find, pax_machine *p) {
4620 19228 site_def const *site = init_noop(find, p);
4621 19228 pax_msg *clone = clone_pax_msg(p->proposer.msg);
4622
1/2
✓ Branch 0 taken 19228 times.
✗ Branch 1 not taken.
 19228 if (clone != nullptr) {
4623 IFDBG(D_CONS, FN; SYCEXP(find));
4624 19228 push_msg_3p(site, p, clone, find, no_op);
4625 } else {
4626 /* purecov: begin inspected */
4627 ✗ G_DEBUG("Unable to propose NoOp due to an OOM error.");
4628 /* purecov: end */
4629 }
4630 19228 }
4631
4632 #if 0
4633 static void propose_noop_2p(synode_no find, pax_machine *p) {
4634 site_def const *site = init_noop(find, p);
4635 IFDBG(D_CONS, FN; SYCEXP(find));
4636 push_msg_2p(site, p);
4637 }
4638 #endif
4639
4640 436444 static void send_read(synode_no find) {
4641 /* Prepare to send a read_op */
4642 436444 site_def const *site = find_site_def(find);
4643
4644 IFDBG(D_NONE, FN; NDBG(get_maxnodes(site), u); NDBG(get_nodeno(site), u););
4645 ADD_DBG(D_CONS, add_event(EVENT_DUMP_PAD, string_arg("find"));
4646 add_synode_event(find); add_event(EVENT_DUMP_PAD, string_arg("site"));
4647 add_event(EVENT_DUMP_PAD, void_arg((void *)find_site_def_rw(find)));
4648 add_event(EVENT_DUMP_PAD, string_arg("get_nodeno(site)"));
4649 add_event(EVENT_DUMP_PAD, uint_arg(get_nodeno(site))););
4650
4651 /* See if node number matches ours */
4652
2/2
✓ Branch 0 taken 436131 times.
✓ Branch 1 taken 313 times.
 436444 if (site) {
4653
2/2
✓ Branch 0 taken 277535 times.
✓ Branch 1 taken 158596 times.
 436131 if (find.node != get_nodeno(site)) {
4654
1/2
✓ Branch 0 taken 277535 times.
✗ Branch 1 not taken.
 277535 pax_msg *pm = pax_msg_new(find, site);
4655
1/2
✓ Branch 0 taken 277535 times.
✗ Branch 1 not taken.
 277535 ref_msg(pm);
4656
1/2
✓ Branch 0 taken 277535 times.
✗ Branch 1 not taken.
 277535 create_read(site, pm);
4657 IFDBG(D_NONE, FN; SYCEXP(find););
4658
4659 IFDBG(D_NONE, FN; NDBG(get_maxnodes(site), u); NDBG(get_nodeno(site), u);
4660 PTREXP(pm));
4661 /* send_server_msg(site, find.node, pm); */
4662 #if 0
4663 send_to_others(site, pm, "send_read");
4664 #else
4665 /* If we have no node number, ask all the others */
4666
3/4
✓ Branch 0 taken 277535 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 86423 times.
✓ Branch 3 taken 191112 times.
 277535 if (get_nodeno(site) == VOID_NODE_NO)
4667
1/2
✓ Branch 0 taken 86423 times.
✗ Branch 1 not taken.
 86423 send_to_others(site, pm, "send_read");
4668 else
4669 /* Ask a random node */
4670
1/2
✓ Branch 0 taken 191112 times.
✗ Branch 1 not taken.
 191112 send_to_someone(site, pm, "send_read");
4671 #endif
4672
1/2
✓ Branch 0 taken 277535 times.
✗ Branch 1 not taken.
 277535 unref_msg(&pm);
4673 } else { /* If node number matches our own number, ask all the others */
4674
1/2
✓ Branch 0 taken 158596 times.
✗ Branch 1 not taken.
 158596 pax_msg *pm = pax_msg_new(find, site);
4675
1/2
✓ Branch 0 taken 158596 times.
✗ Branch 1 not taken.
 158596 ref_msg(pm);
4676
1/2
✓ Branch 0 taken 158596 times.
✗ Branch 1 not taken.
 158596 create_read(site, pm);
4677
1/2
✓ Branch 0 taken 158596 times.
✗ Branch 1 not taken.
 158596 send_to_others(site, pm, "send_read");
4678
1/2
✓ Branch 0 taken 158596 times.
✗ Branch 1 not taken.
 158596 unref_msg(&pm);
4679 }
4680 }
4681 436444 }
4682
4683 /* Find missing values */
4684
4685 59450 static int ok_to_propose(pax_machine *p) {
4686
6/6
✓ Branch 0 taken 54837 times.
✓ Branch 1 taken 4613 times.
✓ Branch 2 taken 25762 times.
✓ Branch 3 taken 29075 times.
✓ Branch 4 taken 19852 times.
✓ Branch 5 taken 10523 times.
 79302 int retval = (is_forcing_node(p) || !recently_active(p)) && !finished(p) &&
4687
2/2
✓ Branch 0 taken 19228 times.
✓ Branch 1 taken 624 times.
 19852 !is_busy_machine(p);
4688 IFDBG(D_NONE, FN; NDBG(p->synode.node, u); NDBG(recently_active(p), d);
4689 NDBG(finished(p), d); NDBG(is_busy_machine(p), d); NDBG(retval, d));
4690 59450 return retval;
4691 }
4692
4693 278735 static void read_missing_values(int n) {
4694 278735 synode_no find = executed_msg;
4695 278735 synode_no end = max_synode;
4696 278735 int i = 0;
4697
4698 IFDBG(D_NONE, FN; SYCEXP(find); SYCEXP(end));
4699
5/6
✓ Branch 0 taken 278735 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 55227 times.
✓ Branch 3 taken 223508 times.
✓ Branch 4 taken 223508 times.
✓ Branch 5 taken 55227 times.
 333962 if (synode_gt(executed_msg, max_synode) ||
4700
2/4
✓ Branch 0 taken 55227 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 55227 times.
 55227 synode_eq(executed_msg, null_synode))
4701 223508 return;
4702
4703
9/12
✓ Branch 0 taken 686762 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 654527 times.
✓ Branch 3 taken 32235 times.
✓ Branch 4 taken 654527 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 654527 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 631535 times.
✓ Branch 9 taken 22992 times.
✓ Branch 10 taken 631535 times.
✓ Branch 11 taken 55227 times.
 686762 while (!synode_gt(find, end) && i < n && !too_far(find)) {
4704
1/2
✓ Branch 0 taken 631535 times.
✗ Branch 1 not taken.
 631535 pax_machine *p = force_get_cache(find);
4705 ADD_DBG(D_NONE, add_synode_event(find); add_synode_event(end);
4706 add_event(EVENT_DUMP_PAD, string_arg("active "));
4707 add_event(EVENT_DUMP_PAD, int_arg(recently_active(p)));
4708 add_event(EVENT_DUMP_PAD, string_arg("finished "));
4709 add_event(EVENT_DUMP_PAD, int_arg(finished(p)));
4710 add_event(EVENT_DUMP_PAD, string_arg("busy "));
4711 add_event(EVENT_DUMP_PAD, int_arg(is_busy_machine(p))););
4712 IFDBG(D_NONE, FN; SYCEXP(find); SYCEXP(end); NDBG(recently_active(p), d);
4713 NDBG(finished(p), d); NDBG(is_busy_machine(p), d));
4714
10/12
✓ Branch 0 taken 631535 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 441775 times.
✓ Branch 3 taken 189760 times.
✓ Branch 4 taken 437371 times.
✓ Branch 5 taken 4404 times.
✓ Branch 6 taken 437371 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 436127 times.
✓ Branch 9 taken 1244 times.
✓ Branch 10 taken 436127 times.
✓ Branch 11 taken 195408 times.
 631535 if (!recently_active(p) && !finished(p) && !is_busy_machine(p)) {
4715
1/2
✓ Branch 0 taken 436127 times.
✗ Branch 1 not taken.
 436127 send_read(find);
4716 }
4717
1/2
✓ Branch 0 taken 631535 times.
✗ Branch 1 not taken.
 631535 find = incr_synode(find);
4718 631535 i++;
4719 }
4720 }
4721
4722 81362 static void propose_missing_values(int n) {
4723 81362 synode_no find = executed_msg;
4724 81362 synode_no end = max_synode;
4725 81362 int i = 0;
4726
4727 IFDBG(D_NONE, FN; NDBG(get_maxnodes(get_site_def()), u); SYCEXP(find);
4728 SYCEXP(end));
4729
5/6
✓ Branch 0 taken 81362 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 5954 times.
✓ Branch 3 taken 75408 times.
✓ Branch 4 taken 75408 times.
✓ Branch 5 taken 5954 times.
 87316 if (synode_gt(executed_msg, max_synode) ||
4730
2/4
✓ Branch 0 taken 5954 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 5954 times.
 5954 synode_eq(executed_msg, null_synode))
4731 75408 return;
4732
4733 IFDBG(D_NONE, FN; SYCEXP(find); SYCEXP(end));
4734 5954 i = 0;
4735
9/12
✓ Branch 0 taken 65790 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 62151 times.
✓ Branch 3 taken 3639 times.
✓ Branch 4 taken 62151 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 62151 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 59837 times.
✓ Branch 9 taken 2314 times.
✓ Branch 10 taken 59837 times.
✓ Branch 11 taken 5953 times.
 65790 while (!synode_gt(find, end) && i < n && !too_far(find)) {
4736
1/2
✓ Branch 0 taken 59837 times.
✗ Branch 1 not taken.
 59837 pax_machine *p = force_get_cache(find);
4737
2/2
✓ Branch 0 taken 1798 times.
✓ Branch 1 taken 58039 times.
 59837 if (wait_forced_config) {
4738 1798 force_pax_machine(p, 1);
4739 }
4740 IFDBG(D_NONE, FN; NDBG(ok_to_propose(p), d); TIMECEXP(task_now());
4741 TIMECEXP(p->last_modified); SYCEXP(find));
4742
1/2
✓ Branch 0 taken 59837 times.
✗ Branch 1 not taken.
 59837 site_def *site = find_site_def_rw(find);
4743
3/4
✓ Branch 0 taken 59837 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 times.
✓ Branch 3 taken 59836 times.
 59837 if (get_nodeno(site) == VOID_NODE_NO) break;
4744
5/6
✓ Branch 0 taken 59836 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 59450 times.
✓ Branch 3 taken 386 times.
✓ Branch 4 taken 19228 times.
✓ Branch 5 taken 40608 times.
 119286 if (!ignore_message(find, site, "propose_missing_values") &&
4745
3/4
✓ Branch 0 taken 59450 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 19228 times.
✓ Branch 3 taken 40222 times.
 59450 ok_to_propose(p)) {
4746
1/2
✓ Branch 0 taken 19228 times.
✗ Branch 1 not taken.
 19228 propose_noop(find, p);
4747 }
4748
1/2
✓ Branch 0 taken 59836 times.
✗ Branch 1 not taken.
 59836 find = incr_synode(find);
4749 59836 i++;
4750 }
4751 }
4752
4753 /* Message handlers */
4754
4755 /*
4756 Reply to the sender of a message.
4757 Avoid using the outbound TCP connection to the node that sent the message, since
4758 it is simpler and safer to always use the same TCP connection as the one the
4759 message arrived on. We then know that the answer will always go to the same
4760 client (and the same instance of that client) that sent the request.
4761 */
4762 #define reply_msg(m) \
4763 { \
4764 if (is_local_node((m)->from, site)) { \
4765 dispatch_op(site, m, NULL); \
4766 } else { \
4767 link_into(&(msg_link_new((m), (m)->from)->l), reply_queue); \
4768 } \
4769 }
4770
4771 #define CREATE_REPLY(x) \
4772 pax_msg *reply = NULL; \
4773 CLONE_PAX_MSG(reply, x)
4774
4775 #define SEND_REPLY \
4776 reply_msg(reply); \
4777 replace_pax_msg(&reply, NULL)
4778
4779 300818 bool_t safe_app_data_copy(pax_msg **target, app_data_ptr source) {
4780 300818 copy_app_data(&(*target)->a, source);
4781
3/4
✓ Branch 0 taken 210519 times.
✓ Branch 1 taken 90299 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 210519 times.
 300818 if ((*target)->a == nullptr && source != nullptr) {
4782 ✗ oom_abort = 1;
4783 ✗ replace_pax_msg(target, nullptr);
4784 ✗ return FALSE;
4785 }
4786 300818 return TRUE;
4787 }
4788
4789 120809 static pax_msg *create_learn_msg_for_ignorant_node(pax_machine *p, pax_msg *pm,
4790 synode_no synode) {
4791
2/4
✓ Branch 0 taken 120809 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 120809 times.
✗ Branch 3 not taken.
 120809 CREATE_REPLY(pm);
4792 IFDBG(D_NONE, FN; SYCEXP(synode));
4793 120809 reply->synode = synode;
4794 120809 reply->proposal = p->learner.msg->proposal;
4795 120809 reply->msg_type = p->learner.msg->msg_type;
4796
1/2
✓ Branch 0 taken 120809 times.
✗ Branch 1 not taken.
 120809 safe_app_data_copy(&reply, p->learner.msg->a);
4797
1/2
✓ Branch 0 taken 120809 times.
✗ Branch 1 not taken.
 120809 if (reply != nullptr) set_learn_type(reply);
4798 /* set_unique_id(reply, p->learner.msg->unique_id); */
4799 120809 return reply;
4800 }
4801
4802 113809 static void teach_ignorant_node(site_def const *site, pax_machine *p,
4803 pax_msg *pm, synode_no synode,
4804 linkage *reply_queue) {
4805
1/2
✓ Branch 0 taken 113809 times.
✗ Branch 1 not taken.
 113809 pax_msg *reply = create_learn_msg_for_ignorant_node(p, pm, synode);
4806
5/12
✓ Branch 0 taken 113809 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 113809 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 113809 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 113809 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 113809 times.
✗ Branch 11 not taken.
 113809 if (reply != nullptr) SEND_REPLY;
4807 113809 }
4808
4809 /* Handle incoming read */
4810 345299 static void handle_read(site_def const *site, pax_machine *p,
4811 linkage *reply_queue, pax_msg *pm) {
4812 IFDBG(D_NONE, FN; BALCEXP(pm->proposal); BALCEXP(p->acceptor.promise);
4813 if (p->acceptor.msg) BALCEXP(p->acceptor.msg->proposal);
4814 STRLIT("type "); STRLIT(pax_msg_type_to_str(pm->msg_type)));
4815
4816
2/2
✓ Branch 0 taken 113809 times.
✓ Branch 1 taken 231490 times.
 345299 if (finished(p)) { /* We have learned a value */
4817 113809 teach_ignorant_node(site, p, pm, pm->synode, reply_queue);
4818 }
4819 345299 }
4820
4821 32125 static pax_msg *create_ack_prepare_msg(pax_machine *p, pax_msg *pm,
4822 synode_no synode) {
4823
2/4
✓ Branch 0 taken 32125 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 32125 times.
✗ Branch 3 not taken.
 32125 CREATE_REPLY(pm);
4824 32125 reply->synode = synode;
4825
2/2
✓ Branch 0 taken 2945 times.
✓ Branch 1 taken 29180 times.
 32125 if (accepted(p)) { /* We have accepted a value */
4826 2945 reply->proposal = p->acceptor.msg->proposal;
4827 2945 reply->msg_type = p->acceptor.msg->msg_type;
4828 IFDBG(D_NONE, FN; STRLIT(" already accepted value "); SYCEXP(synode));
4829 2945 reply->op = ack_prepare_op;
4830
1/2
✓ Branch 0 taken 2945 times.
✗ Branch 1 not taken.
 2945 safe_app_data_copy(&reply, p->acceptor.msg->a);
4831 } else {
4832 IFDBG(D_NONE, FN; STRLIT(" no value synode "); SYCEXP(synode));
4833 29180 reply->op = ack_prepare_empty_op;
4834 }
4835 32125 return reply;
4836 }
4837
4838 34642 pax_msg *handle_simple_prepare(pax_machine *p, pax_msg *pm, synode_no synode) {
4839 34642 pax_msg *reply = nullptr;
4840
2/2
✓ Branch 0 taken 2474 times.
✓ Branch 1 taken 32168 times.
 34642 if (finished(p)) { /* We have learned a value */
4841 IFDBG(D_NONE, FN; SYCEXP(synode); BALCEXP(pm->proposal);
4842 NDBG(finished(p), d));
4843 2474 reply = create_learn_msg_for_ignorant_node(p, pm, synode);
4844 } else {
4845 int greater =
4846 32168 gt_ballot(pm->proposal,
4847 p->acceptor.promise); /* Paxos acceptor phase 1 decision */
4848 IFDBG(D_NONE, FN; SYCEXP(synode); BALCEXP(pm->proposal); NDBG(greater, d));
4849
5/6
✓ Branch 0 taken 43 times.
✓ Branch 1 taken 32125 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 43 times.
✓ Branch 4 taken 32125 times.
✓ Branch 5 taken 43 times.
 32168 if (greater || noop_match(p, pm)) {
4850 32125 p->last_modified = task_now();
4851
1/2
✓ Branch 0 taken 32125 times.
✗ Branch 1 not taken.
 32125 if (greater) {
4852 32125 p->acceptor.promise = pm->proposal; /* promise to not accept any less */
4853 }
4854 32125 reply = create_ack_prepare_msg(p, pm, synode);
4855 }
4856 }
4857 34642 return reply;
4858 }
4859
4860 /* Handle incoming prepare */
4861 34639 static void handle_prepare(site_def const *site, pax_machine *p,
4862 linkage *reply_queue, pax_msg *pm) {
4863 ADD_DBG(D_CONS, add_synode_event(p->synode);
4864 add_event(EVENT_DUMP_PAD, string_arg("pm->from"));
4865 add_event(EVENT_DUMP_PAD, uint_arg(pm->from));
4866 add_event(EVENT_DUMP_PAD, string_arg(pax_op_to_str(pm->op)));
4867 add_event(EVENT_DUMP_PAD, string_arg("proposal"));
4868 add_ballot_event(pm->proposal);
4869 add_event(EVENT_DUMP_PAD, string_arg("promise"));
4870 add_ballot_event(p->acceptor.promise););
4871 IFDBG(D_NONE, FN; BALCEXP(pm->proposal); BALCEXP(p->acceptor.promise);
4872 if (p->acceptor.msg) BALCEXP(p->acceptor.msg->proposal);
4873 STRLIT("type "); STRLIT(pax_msg_type_to_str(pm->msg_type)));
4874
4875 {
4876
1/2
✓ Branch 0 taken 34639 times.
✗ Branch 1 not taken.
 34639 pax_msg *reply = handle_simple_prepare(p, pm, pm->synode);
4877
8/12
✓ Branch 0 taken 34596 times.
✓ Branch 1 taken 43 times.
✓ Branch 2 taken 34596 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 21917 times.
✓ Branch 5 taken 12679 times.
✓ Branch 6 taken 21917 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 12679 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 34639 times.
✗ Branch 11 not taken.
 34639 if (reply != nullptr) SEND_REPLY;
4878 }
4879 34639 }
4880
4881 54400 bool_t check_propose(site_def const *site, pax_machine *p) {
4882 IFDBG(D_NONE, FN; SYCEXP(p->synode);
4883 COPY_AND_FREE_GOUT(dbg_machine_nodeset(p, get_maxnodes(site))););
4884 PAX_MSG_SANITY_CHECK(p->proposer.msg);
4885 {
4886 54400 bool_t can_propose = FALSE;
4887
2/2
✓ Branch 0 taken 19665 times.
✓ Branch 1 taken 34735 times.
 54400 if (prep_majority(site, p)) {
4888 19665 p->proposer.msg->proposal = p->proposer.bal;
4889 19665 BIT_ZERO(p->proposer.prop_nodeset);
4890 19665 p->proposer.msg->synode = p->synode;
4891 19665 init_propose_msg(p->proposer.msg);
4892 19665 p->proposer.sent_prop = p->proposer.bal;
4893 19665 can_propose = TRUE;
4894 }
4895 54400 return can_propose;
4896 }
4897 }
4898
4899 326935 static bool learn_ok(site_def const *site, pax_machine const *p) {
4900
3/4
✓ Branch 0 taken 326935 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 182771 times.
✓ Branch 3 taken 144164 times.
 326935 return get_nodeno(site) != VOID_NODE_NO && prop_majority(site, p);
4901 }
4902
4903 163700 static pax_msg *check_learn(site_def const *site, pax_machine *p) {
4904 IFDBG(D_NONE, FN; SYCEXP(p->synode);
4905 COPY_AND_FREE_GOUT(dbg_machine_nodeset(p, get_maxnodes(site))););
4906 PAX_MSG_SANITY_CHECK(p->proposer.msg);
4907 {
4908 163700 pax_msg *learn_msg = nullptr;
4909
2/2
✓ Branch 0 taken 91543 times.
✓ Branch 1 taken 72157 times.
 163700 if (learn_ok(site, p)) {
4910 91543 p->proposer.msg->synode = p->synode;
4911
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 91543 times.
 91543 if (p->proposer.msg->receivers) free_bit_set(p->proposer.msg->receivers);
4912 91543 p->proposer.msg->receivers = clone_bit_set(p->proposer.prep_nodeset);
4913 91543 BIT_SET(get_nodeno(site), p->proposer.msg->receivers);
4914 if (no_duplicate_payload) {
4915 91543 learn_msg = create_tiny_learn_msg(p, p->proposer.msg);
4916 } else {
4917 /* purecov: begin deadcode */
4918 init_learn_msg(p->proposer.msg);
4919 learn_msg = p->proposer.msg;
4920 /* purecov: end */
4921 }
4922 91543 p->proposer.sent_learn = p->proposer.bal;
4923 }
4924 163700 return learn_msg;
4925 }
4926 }
4927
4928 534033 static void do_learn(site_def const *site [[maybe_unused]], pax_machine *p,
4929 pax_msg *m) {
4930 ADD_DBG(D_CONS, add_synode_event(p->synode);
4931 add_event(EVENT_DUMP_PAD, string_arg("m->from"));
4932 add_event(EVENT_DUMP_PAD, uint_arg(m->from));
4933 add_event(EVENT_DUMP_PAD, string_arg(pax_op_to_str(m->op)));
4934 add_event(EVENT_DUMP_PAD, string_arg("proposal"));
4935 add_ballot_event(m->proposal);
4936 add_event(EVENT_DUMP_PAD, string_arg("promise"));
4937 add_ballot_event(p->acceptor.promise););
4938 /* FN; SYCEXP(p->synode); SYCEXP(m->synode); STRLIT(NEWLINE); */
4939 IFDBG(D_NONE, FN; SYCEXP(p->synode); SYCEXP(m->synode);
4940 dbg_bitset(m->receivers, get_maxnodes(site)););
4941
2/2
✓ Branch 0 taken 163436 times.
✓ Branch 1 taken 370597 times.
 534033 if (m->a) m->a->chosen = TRUE;
4942 534033 replace_pax_msg(&p->acceptor.msg, m);
4943 534033 replace_pax_msg(&p->learner.msg, m);
4944 /*
4945 Track memory used by client data in the cache.
4946 If we do not care about instances that are being decided,
4947 it is only necessary to compute the added memory when we
4948 record the outcome of a consensus round.
4949 */
4950 534033 add_cache_size(p);
4951 /* Shrink the cache size if necessary */
4952 534033 shrink_cache();
4953 534033 }
4954
4955 30955 bool_t handle_simple_ack_prepare(site_def const *site, pax_machine *p,
4956 pax_msg *m) {
4957
1/2
✓ Branch 0 taken 30955 times.
✗ Branch 1 not taken.
 30955 if (get_nodeno(site) != VOID_NODE_NO)
4958 30955 BIT_SET(m->from, p->proposer.prep_nodeset);
4959
4960 {
4961 30955 bool_t can_propose = FALSE;
4962
4/4
✓ Branch 0 taken 2842 times.
✓ Branch 1 taken 28113 times.
✓ Branch 2 taken 29 times.
✓ Branch 3 taken 30926 times.
 33797 if (m->op == ack_prepare_op &&
4963
2/2
✓ Branch 0 taken 29 times.
✓ Branch 1 taken 2813 times.
 2842 gt_ballot(m->proposal, p->proposer.msg->proposal)) { /* greater */
4964 29 replace_pax_msg(&p->proposer.msg, m);
4965
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 29 times.
 29 assert(p->proposer.msg);
4966 }
4967
2/2
✓ Branch 0 taken 30867 times.
✓ Branch 1 taken 88 times.
 30955 if (gt_ballot(m->reply_to, p->proposer.sent_prop)) {
4968 30867 can_propose = check_propose(site, p);
4969 }
4970 30955 return can_propose;
4971 }
4972 }
4973
4974 /* Other node has already accepted a value */
4975 32110 static void handle_ack_prepare(site_def const *site, pax_machine *p,
4976 pax_msg *m) {
4977 ADD_DBG(D_CONS, add_synode_event(p->synode);
4978 add_event(EVENT_DUMP_PAD, string_arg("m->from"));
4979 add_event(EVENT_DUMP_PAD, uint_arg(m->from));
4980 add_event(EVENT_DUMP_PAD, string_arg(pax_op_to_str(m->op))););
4981
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 32110 times.
 32110 assert(m);
4982 IFDBG(D_NONE, FN; if (p->proposer.msg) BALCEXP(p->proposer.msg->proposal);
4983 BALCEXP(p->proposer.bal); BALCEXP(m->reply_to);
4984 BALCEXP(p->proposer.sent_prop); SYCEXP(m->synode));
4985 /*
4986 If the node is preparing a Noop for another node's slot, it is possible
4987 that the leader of the slot has since proposed a value. Hence, there is
4988 no need to move forward if we know that the value has been accepted. This
4989 also prevents changing the size of a learned pax_machine, which would
4990 cause inconsistent reporting of memory usage in P_S.
4991 */
4992
2/2
✓ Branch 0 taken 1097 times.
✓ Branch 1 taken 31013 times.
 32110 if (finished(p)) return;
4993
4994
3/4
✓ Branch 0 taken 31013 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 30952 times.
✓ Branch 3 taken 61 times.
 62026 if (m->from != VOID_NODE_NO &&
4995
2/2
✓ Branch 0 taken 30952 times.
✓ Branch 1 taken 61 times.
 31013 eq_ballot(p->proposer.bal, m->reply_to)) { /* answer to my prepare */
4996 30952 bool_t can_propose = handle_simple_ack_prepare(site, p, m);
4997
2/2
✓ Branch 0 taken 9941 times.
✓ Branch 1 taken 21011 times.
 30952 if (can_propose) send_propose_msg(p->proposer.msg);
4998 }
4999 }
5000
5001 /* #define AUTO_MSG(p,synode) {if(!(p)){replace_pax_msg(&(p),
5002 * pax_msg_new(synode, site));} */
5003
5004 179841 static pax_msg *create_ack_accept_msg(pax_msg *m, synode_no synode) {
5005
2/4
✓ Branch 0 taken 179841 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 179841 times.
✗ Branch 3 not taken.
 179841 CREATE_REPLY(m);
5006 179841 reply->op = ack_accept_op;
5007 179841 reply->synode = synode;
5008 179841 return reply;
5009 }
5010
5011 184367 pax_msg *handle_simple_accept(pax_machine *p, pax_msg *m, synode_no synode) {
5012 184367 pax_msg *reply = nullptr;
5013
2/2
✓ Branch 0 taken 4526 times.
✓ Branch 1 taken 179841 times.
 184367 if (finished(p)) { /* We have learned a value */
5014 4526 reply = create_learn_msg_for_ignorant_node(p, m, synode);
5015 179841 } else if (!gt_ballot(p->acceptor.promise,
5016
2/6
✗ Branch 0 not taken.
✓ Branch 1 taken 179841 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 179841 times.
✗ Branch 5 not taken.
 179841 m->proposal) || /* Paxos acceptor phase 2 decision */
5017 ✗ noop_match(p, m)) {
5018 IFDBG(D_NONE, FN; SYCEXP(m->synode); STRLIT("accept ");
5019 BALCEXP(m->proposal));
5020 179841 p->last_modified = task_now();
5021 179841 replace_pax_msg(&p->acceptor.msg, m);
5022 179841 reply = create_ack_accept_msg(m, synode);
5023 }
5024 184367 return reply;
5025 }
5026
5027 /* Accecpt value if promise is not greater */
5028 184362 static void handle_accept(site_def const *site, pax_machine *p,
5029 linkage *reply_queue, pax_msg *m) {
5030 IFDBG(D_NONE, FN; BALCEXP(p->acceptor.promise); BALCEXP(m->proposal);
5031 STREXP(pax_msg_type_to_str(m->msg_type)));
5032 PAX_MSG_SANITY_CHECK(m);
5033 ADD_DBG(D_CONS, add_synode_event(p->synode);
5034 add_event(EVENT_DUMP_PAD, string_arg("m->from"));
5035 add_event(EVENT_DUMP_PAD, uint_arg(m->from));
5036 add_event(EVENT_DUMP_PAD, string_arg(pax_op_to_str(m->op)));
5037 add_event(EVENT_DUMP_PAD, string_arg("proposal"));
5038 add_ballot_event(m->proposal);
5039 add_event(EVENT_DUMP_PAD, string_arg("promise"));
5040 add_ballot_event(p->acceptor.promise););
5041
5042 {
5043
1/2
✓ Branch 0 taken 184362 times.
✗ Branch 1 not taken.
 184362 pax_msg *reply = handle_simple_accept(p, m, m->synode);
5044
1/2
✓ Branch 0 taken 184362 times.
✗ Branch 1 not taken.
 184362 if (reply != nullptr) {
5045
6/10
✓ Branch 0 taken 184362 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 93579 times.
✓ Branch 3 taken 90783 times.
✓ Branch 4 taken 93579 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 90783 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 184362 times.
✗ Branch 9 not taken.
 184362 SEND_REPLY;
5046 IFDBG(D_CONS, FN; STRLIT("activating sweeper on accept of ");
5047 SYCEXP(m->synode));
5048
1/2
✓ Branch 0 taken 184362 times.
✗ Branch 1 not taken.
 184362 activate_sweeper();
5049 }
5050 }
5051 184362 }
5052
5053 /* Handle answer to accept */
5054 179725 pax_msg *handle_simple_ack_accept(site_def const *site, pax_machine *p,
5055 pax_msg *m) {
5056 179725 pax_msg *learn_msg = nullptr;
5057
4/6
✓ Branch 0 taken 179725 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 179725 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 179711 times.
✓ Branch 5 taken 14 times.
 359450 if (get_nodeno(site) != VOID_NODE_NO && m->from != VOID_NODE_NO &&
5058
2/2
✓ Branch 0 taken 179711 times.
✓ Branch 1 taken 14 times.
 179725 eq_ballot(p->proposer.bal, m->reply_to)) { /* answer to my accept */
5059 179711 BIT_SET(m->from, p->proposer.prop_nodeset);
5060
2/2
✓ Branch 0 taken 163700 times.
✓ Branch 1 taken 16011 times.
 179711 if (gt_ballot(m->proposal, p->proposer.sent_learn)) {
5061 163700 learn_msg = check_learn(site, p);
5062 }
5063 }
5064 179725 return learn_msg;
5065 }
5066 179720 static void handle_ack_accept(site_def const *site, pax_machine *p,
5067 pax_msg *m) {
5068 ADD_DBG(D_CONS, add_synode_event(p->synode);
5069 add_event(EVENT_DUMP_PAD, string_arg("m->from"));
5070 add_event(EVENT_DUMP_PAD, uint_arg(m->from));
5071 add_event(EVENT_DUMP_PAD, string_arg(pax_op_to_str(m->op))););
5072 IFDBG(D_NONE, FN; SYCEXP(m->synode); BALCEXP(p->proposer.bal);
5073 BALCEXP(p->proposer.sent_learn); BALCEXP(m->proposal);
5074 BALCEXP(m->reply_to););
5075 IFDBG(D_NONE, FN; SYCEXP(p->synode);
5076 if (p->acceptor.msg) BALCEXP(p->acceptor.msg->proposal);
5077 BALCEXP(p->proposer.bal); BALCEXP(m->reply_to););
5078
5079 {
5080 179720 pax_msg *learn_msg = handle_simple_ack_accept(site, p, m);
5081
2/2
✓ Branch 0 taken 91541 times.
✓ Branch 1 taken 88179 times.
 179720 if (learn_msg != nullptr) {
5082
1/2
✓ Branch 0 taken 91541 times.
✗ Branch 1 not taken.
 91541 if (learn_msg->op == tiny_learn_op) {
5083 91541 send_tiny_learn_msg(site, learn_msg);
5084 } else {
5085 /* purecov: begin deadcode */
5086 ✗ assert(learn_msg->op == learn_op);
5087 ✗ send_learn_msg(site, learn_msg);
5088 /* purecov: end */
5089 }
5090 }
5091 }
5092 179720 }
5093
5094 /* Handle incoming learn. */
5095 static void activate_sweeper();
5096 161693 void handle_tiny_learn(site_def const *site, pax_machine *pm, pax_msg *p) {
5097
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 161693 times.
 161693 assert(p->msg_type != no_op);
5098
2/2
✓ Branch 0 taken 161376 times.
✓ Branch 1 taken 317 times.
 161693 if (pm->acceptor.msg) {
5099 /* BALCEXP(pm->acceptor.msg->proposal); */
5100
1/2
✓ Branch 0 taken 161376 times.
✗ Branch 1 not taken.
 161376 if (eq_ballot(pm->acceptor.msg->proposal, p->proposal)) {
5101 161376 pm->acceptor.msg->op = learn_op;
5102 161376 pm->last_modified = task_now();
5103 161376 update_max_synode(p);
5104 161376 paxos_fsm(pm, site, paxos_learn, p);
5105 161376 handle_learn(site, pm, pm->acceptor.msg);
5106 } else {
5107 ✗ send_read(p->synode);
5108 IFDBG(D_NONE, FN; STRLIT("tiny_learn"); SYCEXP(p->synode);
5109 BALCEXP(pm->acceptor.msg->proposal); BALCEXP(p->proposal));
5110 }
5111 } else {
5112 317 send_read(p->synode);
5113 IFDBG(D_NONE, FN; STRLIT("tiny_learn"); SYCEXP(p->synode);
5114 BALCEXP(p->proposal));
5115 }
5116 161693 }
5117
5118 4470 static void force_pax_machine(pax_machine *p, int enforcer) {
5119
2/2
✓ Branch 0 taken 1667 times.
✓ Branch 1 taken 2803 times.
 4470 if (!p->enforcer) { /* Not if already marked as forcing node */
5120
2/2
✓ Branch 0 taken 1441 times.
✓ Branch 1 taken 226 times.
 1667 if (enforcer) { /* Only if forcing node */
5121 /* Increase ballot count with a large increment without overflowing */
5122 /* p->proposer.bal.cnt may be -1. */
5123 1441 int32_t delta = (INT32_MAX - MAX(p->proposer.bal.cnt, 0)) / 3;
5124 1441 p->proposer.bal.cnt += delta;
5125 }
5126 }
5127 4470 p->force_delivery = 1;
5128 4470 p->enforcer = enforcer;
5129 4470 }
5130
5131 /* Configure all messages in interval start, end to be forced */
5132 57 static void force_interval(synode_no start, synode_no end, int enforcer) {
5133
2/2
✓ Branch 0 taken 2280 times.
✓ Branch 1 taken 57 times.
 2337 while (!synode_gt(start, end)) {
5134 2280 pax_machine *p = get_cache(start);
5135
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2280 times.
 2280 if (get_nodeno(find_site_def(start)) == VOID_NODE_NO) break;
5136
5137 /* The forcing node will call force_interval twice, first when
5138 the new config is originally installed, and again when it
5139 receives it as an xcom message. start may be the same, but
5140 end will be greater the second time, since it is calculated
5141 based on the message number of the incoming config. Since the forcing
5142 node is the one responsible for delivering all messages until the
5143 start of the new site, it is important that all instances belonging to
5144 the old site are correctly marked. */
5145
5146
2/2
✓ Branch 0 taken 618 times.
✓ Branch 1 taken 1662 times.
 2280 if (p->enforcer) enforcer = 1; /* Extend to new instances */
5147 2280 force_pax_machine(p, enforcer);
5148
5149 /* Old nodesets are null and void */
5150 2280 BIT_ZERO(p->proposer.prep_nodeset);
5151 2280 BIT_ZERO(p->proposer.prop_nodeset);
5152 2280 start = incr_synode(start);
5153 }
5154 57 }
5155
5156 57 static void start_force_config(site_def *s, int enforcer) {
5157
1/2
✓ Branch 0 taken 57 times.
✗ Branch 1 not taken.
 57 synode_no end = add_event_horizon(s->boot_key);
5158
5159 IFDBG(D_NONE, FN; SYCEXP(executed_msg); SYCEXP(end));
5160
4/6
✓ Branch 0 taken 57 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 27 times.
✓ Branch 3 taken 30 times.
✓ Branch 4 taken 27 times.
✗ Branch 5 not taken.
 57 if (synode_gt(end, max_synode)) set_max_synode(end);
5161
5162
1/2
✓ Branch 0 taken 57 times.
✗ Branch 1 not taken.
 57 free_forced_config_site_def();
5163 57 wait_forced_config = 0;
5164 57 forced_config = s;
5165
1/2
✓ Branch 0 taken 57 times.
✗ Branch 1 not taken.
 57 force_interval(executed_msg, max_synode,
5166 enforcer); /* Force everything in the pipeline */
5167 57 }
5168
5169 /* Learn this value */
5170 441191 void handle_learn(site_def const *site, pax_machine *p, pax_msg *m) {
5171 IFDBG(D_NONE, FN; STRLIT("proposer nodeset ");
5172 dbg_bitset(p->proposer.prop_nodeset, get_maxnodes(site)););
5173 IFDBG(D_NONE, FN; STRLIT("receivers ");
5174 dbg_bitset(m->receivers, get_maxnodes(site)););
5175 IFDBG(D_NONE, FN; NDBG(task_now(), f); SYCEXP(p->synode);
5176 COPY_AND_FREE_GOUT(dbg_app_data(m->a)););
5177
5178 PAX_MSG_SANITY_CHECK(m);
5179 441191 p->last_modified = task_now();
5180
2/2
✓ Branch 0 taken 285874 times.
✓ Branch 1 taken 155317 times.
 441191 if (!finished(p)) { /* Avoid re-learn */
5181 285874 activate_sweeper();
5182 285874 do_learn(site, p, m);
5183 /* Check for special messages */
5184
4/4
✓ Branch 0 taken 163436 times.
✓ Branch 1 taken 122438 times.
✓ Branch 2 taken 848 times.
✓ Branch 3 taken 162588 times.
 285874 if (m->a && m->a->body.c_t == unified_boot_type) {
5185 IFDBG(D_NONE, FN; STRLIT("Got unified_boot "); SYCEXP(p->synode);
5186 SYCEXP(m->synode););
5187
4/4
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 847 times.
✓ Branch 2 taken 1 times.
✓ Branch 3 taken 847 times.
 848 XCOM_FSM(x_fsm_net_boot, void_arg(m->a));
5188 }
5189 /* See if someone is forcing a new config */
5190
3/4
✓ Branch 0 taken 82 times.
✓ Branch 1 taken 285792 times.
✓ Branch 2 taken 82 times.
✗ Branch 3 not taken.
 285874 if (m->force_delivery && m->a) {
5191 IFDBG(D_NONE, FN; STRLIT("Got forced config "); SYCEXP(p->synode);
5192 SYCEXP(m->synode););
5193 /* Configure all messages from executed_msg until start of new config
5194 as forced messages so they will eventually be finished */
5195 /* Immediately install this new config */
5196
2/4
✗ Branch 0 not taken.
✗ Branch 1 not taken.
✓ Branch 2 taken 30 times.
✓ Branch 3 taken 52 times.
 82 switch (m->a->body.c_t) {
5197 ✗ case add_node_type:
5198 /* purecov: begin deadcode */
5199 ✗ if (should_ignore_forced_config_or_view(
5200 ✗ find_site_def(p->synode)->x_proto)) {
5201 ✗ log_ignored_forced_config(m->a, "handle_learn");
5202 } else {
5203 ✗ site_def *new_def = handle_add_node(m->a);
5204 ✗ if (new_def) start_force_config(clone_site_def(new_def), 0);
5205 }
5206 ✗ break;
5207 /* purecov: end */
5208 ✗ case remove_node_type:
5209 /* purecov: begin deadcode */
5210 ✗ if (should_ignore_forced_config_or_view(
5211 ✗ find_site_def(p->synode)->x_proto)) {
5212 ✗ log_ignored_forced_config(m->a, "handle_learn");
5213 } else {
5214 ✗ start_force_config(clone_site_def(handle_remove_node(m->a)), 0);
5215 }
5216 ✗ break;
5217 /* purecov: end */
5218 30 case force_config_type:
5219 30 start_force_config(clone_site_def(install_node_group(m->a)), 0);
5220 30 break;
5221 52 default:
5222 52 break;
5223 }
5224 }
5225 }
5226
5227 441191 task_wakeup(&p->rv);
5228 441191 }
5229
5230 /* Skip this value */
5231 249014 static void handle_skip(site_def const *site, pax_machine *p, pax_msg *m) {
5232 /* IFDBG(D_NONE, FN;); */
5233 /* IFDBG(D_NONE, FN; NDBG(task_now(),f); SYCEXP(p->msg->synode)); */
5234
2/2
✓ Branch 0 taken 248159 times.
✓ Branch 1 taken 855 times.
 249014 if (!finished(p)) {
5235 248159 p->last_modified = task_now();
5236 248159 skip_value(m);
5237 248159 do_learn(site, p, m);
5238 }
5239 /* IFDBG(D_NONE, FN; STRLIT("taskwakeup "); SYCEXP(p->msg->synode)); */
5240 249014 task_wakeup(&p->rv);
5241 249014 }
5242
5243 80447 static void handle_client_msg(pax_msg *p) {
5244
2/4
✓ Branch 0 taken 80447 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 80447 times.
 80447 if (!p || p->a == nullptr) /* discard invalid message */
5245 ✗ return;
5246 {
5247 80447 msg_link *ml = msg_link_new(p, VOID_NODE_NO);
5248
5249 /* Put it in the proposer queue */
5250 ADD_T_EV(task_now(), __FILE__, __LINE__, "handle_client_msg");
5251 80447 channel_put(&prop_input_queue, &ml->l);
5252 }
5253 }
5254
5255 #ifdef ACCEPT_SITE_TEST
5256 /* See if we should process an incoming ping from a node.
5257 The purpose is to avoid doing recovery from a node with an obsolete site
5258 definition */
5259 static int accept_site(site_def const *site) {
5260 site_def *mysite = (site_def *)get_site_def();
5261
5262 if (site) {
5263 if (!mysite) {
5264 site_def *prev = (site_def *)find_prev_site_def(site->boot_key);
5265 IFDBG(
5266 D_NONE, FN; PTREXP(site); PTREXP(mysite); PTREXP(prev);
5267 SYCEXP(site->boot_key); if (prev) { SYCEXP(prev->boot_key); });
5268 if (!prev) {
5269 /** alive when no site, no known previous definition, and present in
5270 * new is accepted */
5271 return (site->boot_key.group_id == 0
5272 ? 1
5273 : (xcom_find_node_index((node_list *)&site->nodes) !=
5274 VOID_NODE_NO));
5275 } else {
5276 /** alive when no site, a previous definition of groupid is known, but
5277 * is older than site def, is accepted */
5278 return synode_gt(site->boot_key, prev->boot_key);
5279 }
5280 } else {
5281 IFDBG(D_NONE, FN; PTREXP(site); PTREXP(mysite); SYCEXP(site->boot_key);
5282 SYCEXP(mysite->boot_key));
5283 if (get_group_id(site) != get_group_id(mysite)) {
5284 /** alive from different site should never be accepted */
5285 return 0;
5286 } else {
5287 /** alive from same site should be accepted if boot_key is larger than
5288 * mine */
5289 node_no my_nodeno = xcom_find_node_index((node_list *)&mysite->nodes);
5290 node_no site_nodeno = xcom_find_node_index((node_list *)&site->nodes);
5291 return (synode_gt(site->boot_key, mysite->boot_key) &&
5292 ((my_nodeno != VOID_NODE_NO) || (site_nodeno != VOID_NODE_NO)));
5293 }
5294 }
5295 }
5296 /** Always accept a NULL site */
5297 IFDBG(D_NONE, FN; PTREXP(site));
5298 return 1;
5299 }
5300 #endif
5301
5302 /* Handle incoming "need boot" message. */
5303 /* purecov: begin deadcode */
5304 ✗ static inline void handle_boot(site_def const *site, linkage *reply_queue,
5305 pax_msg *p) {
5306 /* This should never be TRUE, but validate it instead of asserting. */
5307 ✗ if (site == nullptr || site->nodes.node_list_len < 1) {
5308 ✗ G_DEBUG(
5309 "handle_boot: Received an unexpected need_boot_op when site == NULL "
5310 "or "
5311 "site->nodes.node_list_len < 1");
5312 ✗ return;
5313 }
5314
5315 ✗ if (ALWAYS_HANDLE_NEED_BOOT || should_handle_need_boot(site, p)) {
5316 ✗ handle_need_snapshot(reply_queue, p);
5317 } else {
5318 ✗ G_DEBUG(
5319 "Ignoring a need_boot_op message from an XCom incarnation that does "
5320 "not belong to the group.");
5321 }
5322 }
5323 /* purecov: end */
5324
5325 1322 bool_t should_handle_need_boot(site_def const *site, pax_msg *p) {
5326 1322 bool_t should_handle = FALSE;
5327 1322 bool_t const sender_advertises_identity =
5328
3/4
✓ Branch 0 taken 1321 times.
✓ Branch 1 taken 1 times.
✓ Branch 2 taken 1321 times.
✗ Branch 3 not taken.
 1322 (p->a != nullptr && p->a->body.c_t == xcom_boot_type);
5329
5330 /*
5331 If the message advertises the sender's identity, check if it matches the
5332 membership information.
5333
5334 The sender's identity may not match if, e.g.:
5335
5336 a. The member was already removed, or
5337 b. It is a new incarnation of a crashed member that is yet to be removed.
5338
5339 ...or some other reason.
5340
5341 If it is due to reason (b), we do not want to boot the sender because XCom
5342 only implements a simple fail-stop model. Allowing the sender to rejoin the
5343 group without going through the full remove+add node path could violate
5344 safety because the sender does not remember any previous Paxos acceptances
5345 it acknowledged before crashing. Since the pre-crash incarnation may have
5346 accepted a value for a given synod but the post-crash incarnation has
5347 forgotten that fact, the post-crash incarnation will fail to propagate the
5348 previously accepted value to a higher ballot. Since majorities can overlap
5349 on a single node, if the overlap node is the post-crash incarnation which
5350 has forgotten about the previously accepted value, a higher ballot proposer
5351 may get a different value accepted, leading to conflicting values to be
5352 accepted for different proposers, which is a violation of the safety
5353 properties of the Paxos protocol.
5354
5355 If the sender does not advertise its identity, we boot it unconditionally.
5356 This is for backwards compatibility.
5357 */
5358
2/2
✓ Branch 0 taken 1321 times.
✓ Branch 1 taken 1 times.
 1322 if (sender_advertises_identity) {
5359 1321 bool_t const sender_advertises_one_identity =
5360 1321 (p->a->body.app_u_u.nodes.node_list_len == 1);
5361
5362 /* Defensively accept only messages with a single identity. */
5363
2/2
✓ Branch 0 taken 1320 times.
✓ Branch 1 taken 1 times.
 1321 if (sender_advertises_one_identity) {
5364 1320 node_address *sender_identity = p->a->body.app_u_u.nodes.node_list_val;
5365
5366 1320 should_handle = node_exists_with_uid(sender_identity, &site->nodes);
5367 }
5368 } else {
5369 1 should_handle = TRUE;
5370 }
5371
5372 1322 return should_handle;
5373 }
5374
5375 1370 void init_need_boot_op(pax_msg *p, node_address *identity) {
5376 1370 p->op = need_boot_op;
5377
2/2
✓ Branch 0 taken 1369 times.
✓ Branch 1 taken 1 times.
 1370 if (identity != nullptr) {
5378 1369 p->a = new_app_data();
5379 1369 p->a->body.c_t = xcom_boot_type;
5380 1369 init_node_list(1, identity, &p->a->body.app_u_u.nodes);
5381 }
5382 1370 }
5383
5384 #define PING_GATHERING_TIME_WINDOW 5.0
5385 #define PINGS_GATHERED_BEFORE_CONNECTION_SHUTDOWN 3
5386
5387 288303 int pre_process_incoming_ping(site_def const *site, pax_msg const *pm,
5388 int has_client_already_booted,
5389 double current_time) {
5390 // Yes... it is a ping for me, boot is done and it is a are_you_alive_op
5391 // This means that something wrong is not right...
5392 288303 int did_shutdown = 0;
5393
5394
6/6
✓ Branch 0 taken 144196 times.
✓ Branch 1 taken 144107 times.
✓ Branch 2 taken 139991 times.
✓ Branch 3 taken 4205 times.
✓ Branch 4 taken 3495 times.
✓ Branch 5 taken 284808 times.
 428294 if ((pm->from != get_nodeno(site)) && has_client_already_booted &&
5395
2/2
✓ Branch 0 taken 3495 times.
✓ Branch 1 taken 136496 times.
 139991 (pm->op == are_you_alive_op)) {
5396
2/2
✓ Branch 0 taken 17 times.
✓ Branch 1 taken 3478 times.
 3495 G_DEBUG(
5397 "Received a ping to myself. This means that something must be wrong "
5398 "in "
5399 "a bi-directional connection")
5400 // Going to kill the connection for that node...
5401
2/4
✓ Branch 0 taken 3495 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 3495 times.
✗ Branch 3 not taken.
 3495 if (site && (pm->from < site->nodes.node_list_len)) {
5402 // This is not the first ping received in the last 5 seconds...
5403 3495 if (site->servers[pm->from]->last_ping_received >
5404
2/2
✓ Branch 0 taken 1273 times.
✓ Branch 1 taken 2222 times.
 3495 (current_time - PING_GATHERING_TIME_WINDOW)) {
5405 1273 site->servers[pm->from]->number_of_pings_received++;
5406 } else { // First ping since at least more than 5 seconds...
5407 2222 site->servers[pm->from]->number_of_pings_received = 1;
5408 }
5409
5410 3495 site->servers[pm->from]->last_ping_received = current_time;
5411
5412 // If we keep on receiving periodical pings... lets kill the connection
5413
4/4
✓ Branch 0 taken 2532 times.
✓ Branch 1 taken 963 times.
✓ Branch 2 taken 116 times.
✓ Branch 3 taken 3379 times.
 6027 if (is_connected(site->servers[pm->from]->con) &&
5414
2/2
✓ Branch 0 taken 116 times.
✓ Branch 1 taken 2416 times.
 2532 site->servers[pm->from]->number_of_pings_received ==
5415 PINGS_GATHERED_BEFORE_CONNECTION_SHUTDOWN) {
5416 116 shutdown_connection(site->servers[pm->from]->con);
5417
2/4
✓ Branch 0 taken 116 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 116 times.
✗ Branch 3 not taken.
 116 G_WARNING(
5418 "Shutting down an outgoing connection. This happens because "
5419 "something might be wrong on a bi-directional connection to node "
5420 "%s:%d. Please check the connection status to this member",
5421 site->servers[pm->from]->srv, site->servers[pm->from]->port);
5422 116 did_shutdown = 1;
5423 }
5424 }
5425 }
5426
5427 288303 return did_shutdown;
5428 }
5429
5430 /* Handle incoming alive message */
5431 static double sent_alive = 0.0;
5432 288283 static inline void handle_alive(site_def const *site, linkage *reply_queue,
5433 pax_msg *pm) {
5434 288283 pre_process_incoming_ping(site, pm, client_boot_done, task_now());
5435
5436
6/6
✓ Branch 0 taken 4201 times.
✓ Branch 1 taken 284082 times.
✓ Branch 2 taken 2834 times.
✓ Branch 3 taken 1367 times.
✓ Branch 4 taken 286916 times.
✓ Branch 5 taken 1367 times.
 288283 if (client_boot_done || !(task_now() - sent_alive > 1.0)) /* Already done? */
5437 286916 return;
5438
5439 #ifdef ACCEPT_SITE_TEST
5440 if (!accept_site(site)) return;
5441 #endif
5442
5443 /* Avoid responding to own ping */
5444
3/6
✓ Branch 0 taken 1367 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 1367 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 1367 times.
 1367 if (pm->from == get_nodeno(site) || pm->from == pm->to) return;
5445
5446 /*
5447 This code will check if the ping is intended to us.
5448 If the encoded node does not exist in the current configuration,
5449 we avoid sending need_boot_op, since it must be from a different
5450 reincarnation of this node.
5451 */
5452
1/6
✗ Branch 0 not taken.
✓ Branch 1 taken 1367 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 1367 if (site && pm->a && pm->a->body.c_t == xcom_boot_type) {
5453 IFDBG(D_NONE, FN;
5454 COPY_AND_FREE_GOUT(dbg_list(&pm->a->body.app_u_u.nodes)););
5455
5456 ✗ if (!node_exists_with_uid(&pm->a->body.app_u_u.nodes.node_list_val[0],
5457 ✗ &get_site_def()->nodes))
5458 ✗ return;
5459 }
5460
5461
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1367 times.
 1367 if (is_dead_site(pm->group_id)) return; /* Avoid dealing with zombies */
5462
5463 {
5464
2/4
✓ Branch 0 taken 1367 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1367 times.
✗ Branch 3 not taken.
 1367 CREATE_REPLY(pm);
5465
2/4
✓ Branch 0 taken 1367 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1367 times.
✗ Branch 3 not taken.
 1367 init_need_boot_op(reply, cfg_app_xcom_get_identity());
5466
1/2
✓ Branch 0 taken 1367 times.
✗ Branch 1 not taken.
 1367 sent_alive = task_now();
5467
2/4
✓ Branch 0 taken 1367 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1367 times.
✗ Branch 3 not taken.
 1367 G_INFO(
5468 "Node has not booted. Requesting an XCom snapshot from node number %d "
5469 "in the current configuration",
5470 pm->from);
5471
4/10
✓ Branch 0 taken 1367 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 1367 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 1367 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 1367 times.
✗ Branch 9 not taken.
 1367 SEND_REPLY;
5472 }
5473 IFDBG(D_NONE, FN; STRLIT("sent need_boot_op"););
5474 }
5475
5476 442556 static void update_max_synode(pax_msg *p) {
5477
2/2
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 442555 times.
 442556 if (is_dead_site(p->group_id)) return;
5478
5/6
✓ Branch 0 taken 420889 times.
✓ Branch 1 taken 21666 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 420889 times.
✓ Branch 4 taken 21666 times.
✓ Branch 5 taken 420889 times.
 442555 if (get_group_id(get_site_def()) == 0 || max_synode.group_id == 0) {
5479 21666 set_max_synode(p->synode);
5480
1/2
✓ Branch 0 taken 420889 times.
✗ Branch 1 not taken.
 420889 } else if (max_synode.group_id == p->synode.group_id) {
5481
2/2
✓ Branch 0 taken 149622 times.
✓ Branch 1 taken 271267 times.
 420889 if (synode_gt(p->synode, max_synode)) {
5482 149622 set_max_synode(p->synode);
5483 }
5484
2/2
✓ Branch 0 taken 2959 times.
✓ Branch 1 taken 417930 times.
 420889 if (synode_gt(p->max_synode, max_synode)) {
5485 2959 set_max_synode(p->max_synode);
5486 }
5487 }
5488 }
5489
5490 /* Message dispatch */
5491 #define BAL_FMT "ballot {cnt %d node %d}"
5492 #define BAL_MEM(x) (x).cnt, (x).node
5493
5494 static int clicnt = 0;
5495
5496 23 xcom_event_horizon xcom_get_minimum_event_horizon() {
5497 23 return EVENT_HORIZON_MIN;
5498 }
5499
5500 23 xcom_event_horizon xcom_get_maximum_event_horizon() {
5501 23 return EVENT_HORIZON_MAX;
5502 }
5503
5504 /**
5505 * Retrieves the latest event horizon.
5506 *
5507 * There is no specific reason for this method to return the latest event
5508 * horizon instead of the current one. Both would be acceptable results of
5509 * this function, but we had to make a decision of one over the other.
5510 *
5511 * @param[out] event_horizon the latest event horizon
5512 * @retval REQUEST_FAIL XCom is not initialized yet
5513 * @retval REQUEST_OK function was successful and event_horizon contains the
5514 * latest event horizon
5515 */
5516 99 static client_reply_code xcom_get_event_horizon(
5517 xcom_event_horizon *event_horizon) {
5518 99 site_def const *latest_config = get_site_def();
5519
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 99 times.
 99 if (latest_config == nullptr) return REQUEST_FAIL;
5520 99 *event_horizon = latest_config->event_horizon;
5521 99 return REQUEST_OK;
5522 }
5523
5524 1729 static u_int allow_add_node(app_data_ptr a) {
5525 /* Get information on the current site definition */
5526 1729 const site_def *new_site_def = get_site_def();
5527 1729 const site_def *valid_site_def = find_site_def(executed_msg);
5528
5529 /* Get information on the nodes to be added */
5530 1729 u_int nr_nodes_to_add = a->body.app_u_u.nodes.node_list_len;
5531 1729 node_address *nodes_to_change = a->body.app_u_u.nodes.node_list_val;
5532
5533
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1729 times.
 1729 if (check_if_add_node_is_unsafe_against_event_horizon(a)) return 0;
5534
5535
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1729 times.
 1729 if (unsafe_leaders(a)) {
5536 ✗ return 0;
5537 }
5538
5539
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1729 times.
 1729 if (add_node_unsafe_against_ipv4_old_nodes(a)) {
5540 ✗ G_MESSAGE(
5541 "This server is unable to join the group as the NIC used is "
5542 "configured "
5543 "with IPv6 only and there are members in the group that are unable "
5544 "to "
5545 "communicate using IPv6, only IPv4.Please configure this server to "
5546 "join the group using an IPv4 address instead.");
5547 ✗ return 0;
5548 }
5549
5550 {
5551 u_int i;
5552
2/2
✓ Branch 0 taken 1729 times.
✓ Branch 1 taken 1329 times.
 3058 for (i = 0; i < nr_nodes_to_add; i++) {
5553
4/4
✓ Branch 0 taken 1329 times.
✓ Branch 1 taken 400 times.
✓ Branch 2 taken 400 times.
✓ Branch 3 taken 1329 times.
 3058 if (node_exists(&nodes_to_change[i], &new_site_def->nodes) ||
5554
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1329 times.
 1329 node_exists(&nodes_to_change[i], &valid_site_def->nodes)) {
5555 /*
5556 We are simply ignoring the attempt to add a node to the
5557 group when there is an old incarnation of it, meaning
5558 that the node has crashed and restarted so fastly that
5559 nobody has noticed that it has gone.
5560
5561 In XCOM, the group is not automatically reconfigured
5562 and it is possible to start reusing a node that has
5563 crashed and restarted without reconfiguring the group
5564 by adding the node back to it.
5565
5566 However, this operation may be unsafe because XCOM
5567 does not implement a crash-recovery model and nodes
5568 suffer from amnesia after restarting the service. In
5569 other words this may lead to inconsistency issues in
5570 the paxos protocol.
5571
5572 Unfortunately, preventing that a node is added back
5573 to the system where there is an old incarnation will
5574 not fix this problem since other changes are required.
5575 */
5576
2/4
✓ Branch 0 taken 400 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 400 times.
✗ Branch 3 not taken.
 400 G_WARNING(
5577 "Old incarnation found while trying to "
5578 "add node %s %.*s. Please stop the old node or wait for it to "
5579 "leave the group.",
5580 nodes_to_change[i].address, nodes_to_change[i].uuid.data.data_len,
5581 nodes_to_change[i].uuid.data.data_val);
5582 400 return 0;
5583 }
5584 }
5585 }
5586
5587 1329 return 1;
5588 }
5589
5590 2310 static u_int allow_remove_node(app_data_ptr a) {
5591 /* Get information on the current site definition */
5592 2310 const site_def *new_site_def = get_site_def();
5593
5594 /* Get information on the nodes to be added */
5595 2310 u_int nodes_len = a->body.app_u_u.nodes.node_list_len;
5596 2310 node_address *nodes_to_change = a->body.app_u_u.nodes.node_list_val;
5597
5598 u_int i;
5599
2/2
✓ Branch 0 taken 2310 times.
✓ Branch 1 taken 2261 times.
 4571 for (i = 0; i < nodes_len; i++) {
5600
2/2
✓ Branch 0 taken 49 times.
✓ Branch 1 taken 2261 times.
 2310 if (!node_exists_with_uid(&nodes_to_change[i], &new_site_def->nodes)) {
5601 /*
5602 If the UID does not exist, then 1) the node has already been
5603 removed or 2) it has reincarnated.
5604 */
5605 /* purecov: begin inspected */
5606
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 49 times.
 49 if (node_exists(&nodes_to_change[i], &new_site_def->nodes)) {
5607 /*
5608 We also cannot allow an upper-layer to remove a new incarnation
5609 of a node when it tries to remove an old one.
5610 */
5611 ✗ G_MESSAGE(
5612 "New incarnation found while trying to "
5613 "remove node %s %.*s.",
5614 nodes_to_change[i].address, nodes_to_change[i].uuid.data.data_len,
5615 nodes_to_change[i].uuid.data.data_val);
5616 } else {
5617 /* The node has already been removed, so we block the request */
5618
2/4
✓ Branch 0 taken 49 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 49 times.
✗ Branch 3 not taken.
 49 G_MESSAGE(
5619 "Node has already been removed: "
5620 "%s %.*s.",
5621 nodes_to_change[i].address, nodes_to_change[i].uuid.data.data_len,
5622 nodes_to_change[i].uuid.data.data_val);
5623 }
5624 49 return 0;
5625 /* purecov: end */
5626 }
5627 }
5628
5629 2261 return 1;
5630 }
5631
5632 /**
5633 * Logs the fact that an add/remove node request is aimed at another group.
5634 *
5635 * @param a a pointer to the app_data of the configuration command
5636 * @param message_fmt a formatted message to log, containing a single %s that
5637 * will be replaced by the node's address
5638 */
5639 102 static void log_cfgchange_wrong_group(app_data_ptr a,
5640 const char *const message_fmt) {
5641 102 u_int const nr_nodes = a->body.app_u_u.nodes.node_list_len;
5642 u_int i;
5643
2/2
✓ Branch 0 taken 102 times.
✓ Branch 1 taken 102 times.
 204 for (i = 0; i < nr_nodes; i++) {
5644 102 char const *const address = a->body.app_u_u.nodes.node_list_val[i].address;
5645
2/4
✓ Branch 0 taken 102 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 102 times.
✗ Branch 3 not taken.
 102 G_WARNING(message_fmt, address);
5646 }
5647 102 }
5648
5649 /**
5650 * Validates if a configuration command can be executed.
5651 * Checks whether the configuration command is aimed at the correct group.
5652 * Checks whether the configuration command pertains to a node reincarnation.
5653 *
5654 * @param p a pointer to the pax_msg of the configuration command
5655 * @retval REQUEST_OK if the reconfiguration command can be executed
5656 * @retval REQUEST_RETRY if XCom is still booting
5657 * @retval REQUEST_FAIL if the configuration command cannot be executed
5658 */
5659 4217 static client_reply_code can_execute_cfgchange(pax_msg *p) {
5660 4217 app_data_ptr a = p->a;
5661
5662
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4217 times.
 4217 if (executed_msg.msgno <= 2) {
5663 // If we have not booted and we receive an add_node that contains us...
5664 ✗ if (add_node_adding_own_address(a))
5665 ✗ return REQUEST_FAIL;
5666 else {
5667 /*G_INFO(
5668 "Configuration change failed. Request on a node that has not booted "
5669 "yet.");*/
5670 ✗ G_INFO(
5671 "This node received a Configuration change request, but it not yet "
5672 "started. This could happen if one starts several nodes "
5673 "simultaneously. This request will be retried by whoever sent it.");
5674 ✗ return REQUEST_RETRY;
5675 }
5676 }
5677
5678
4/6
✓ Branch 0 taken 4217 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 4217 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 102 times.
✓ Branch 5 taken 4115 times.
 4217 if (a && a->group_id != 0 && a->group_id != executed_msg.group_id) {
5679
1/6
✓ Branch 0 taken 102 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 102 switch (a->body.c_t) {
5680 102 case add_node_type:
5681 102 log_cfgchange_wrong_group(
5682 a,
5683 "The request to add %s to the group has been rejected because it "
5684 "is aimed at another group");
5685 102 break;
5686 ✗ case remove_node_type:
5687 ✗ log_cfgchange_wrong_group(a,
5688 "The request to remove %s from the group "
5689 "has been rejected because "
5690 "it is aimed at another group");
5691 ✗ break;
5692 ✗ case force_config_type:
5693 ✗ G_WARNING(
5694 "The request to force the group membership has been rejected "
5695 "because it is aimed at another group");
5696 ✗ break;
5697 ✗ case set_max_leaders:
5698 ✗ G_WARNING(
5699 "The request to change max number of leaders has been rejected "
5700 "because it is aimed at another group");
5701 ✗ break;
5702 ✗ case set_leaders_type:
5703 ✗ G_WARNING(
5704 "The request to change leaders has been rejected "
5705 "because it is aimed at another group");
5706 ✗ break;
5707 ✗ default:
5708 ✗ assert(0 &&
5709 "A cargo_type different from {add_node_type, remove_node_type, "
5710 "force_config_type, set_max_leaders, set_leaders_type} should "
5711 "not "
5712 "have hit this code path");
5713 }
5714 102 return REQUEST_FAIL;
5715 }
5716
5717
7/8
✓ Branch 0 taken 4115 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1729 times.
✓ Branch 3 taken 2386 times.
✓ Branch 4 taken 400 times.
✓ Branch 5 taken 1329 times.
✓ Branch 6 taken 400 times.
✓ Branch 7 taken 3715 times.
 4115 if (a && a->body.c_t == add_node_type && !allow_add_node(a))
5718 400 return REQUEST_FAIL;
5719
5720
7/8
✓ Branch 0 taken 3715 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2310 times.
✓ Branch 3 taken 1405 times.
✓ Branch 4 taken 49 times.
✓ Branch 5 taken 2261 times.
✓ Branch 6 taken 49 times.
✓ Branch 7 taken 3666 times.
 3715 if (a && a->body.c_t == remove_node_type && !allow_remove_node(a))
5721 49 return REQUEST_FAIL;
5722
5723
4/6
✓ Branch 0 taken 3666 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 9 times.
✓ Branch 3 taken 3657 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 3666 times.
 3675 if (a && a->body.c_t == set_event_horizon_type &&
5724
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 9 times.
 9 is_unsafe_event_horizon_reconfiguration(a))
5725 ✗ return REQUEST_FAIL;
5726
5727
5/6
✓ Branch 0 taken 3666 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 28 times.
✓ Branch 3 taken 3638 times.
✓ Branch 4 taken 1 times.
✓ Branch 5 taken 3665 times.
 3694 if (a && a->body.c_t == force_config_type &&
5728
2/2
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 27 times.
 28 are_there_dead_nodes_in_new_config(a))
5729 1 return REQUEST_FAIL;
5730
5731 7330 if (a &&
5732
5/8
✓ Branch 0 taken 3665 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 3665 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 39 times.
✓ Branch 5 taken 3626 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 3665 times.
 3704 (a->body.c_t == set_max_leaders || a->body.c_t == set_leaders_type) &&
5733
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 39 times.
 39 is_unsafe_leaders_reconfiguration(a))
5734 ✗ return REQUEST_FAIL;
5735
5736 3665 return REQUEST_OK;
5737 }
5738
5739 662651 static void activate_sweeper() {
5740
2/2
✓ Branch 0 taken 616131 times.
✓ Branch 1 taken 46520 times.
 662651 if (sweeper) {
5741 ADD_DBG(D_CONS, add_event(EVENT_DUMP_PAD,
5742 string_arg("sweeper activated max_synode"));
5743 add_synode_event(max_synode););
5744 616131 task_activate(sweeper);
5745 }
5746 662651 }
5747
5748 static synode_no start_config = NULL_SYNODE;
5749
5750 99 void dispatch_get_event_horizon(site_def const *site, pax_msg *p,
5751 linkage *reply_queue) {
5752
2/4
✓ Branch 0 taken 99 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 99 times.
✗ Branch 3 not taken.
 99 CREATE_REPLY(p);
5753 IFDBG(D_NONE, FN; STRLIT("Got get_event_horizon from client");
5754 SYCEXP(p->synode););
5755 99 reply->op = xcom_client_reply;
5756
1/2
✓ Branch 0 taken 99 times.
✗ Branch 1 not taken.
 99 reply->cli_err = xcom_get_event_horizon(&reply->event_horizon);
5757
4/10
✓ Branch 0 taken 99 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 99 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 99 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 99 times.
✗ Branch 9 not taken.
 99 SEND_REPLY;
5758 99 }
5759
5760 85 static reply_data *new_leader_info(site_def *site) {
5761
1/2
✓ Branch 0 taken 85 times.
✗ Branch 1 not taken.
 85 if (site) {
5762 reply_data *data =
5763 85 static_cast<reply_data *>(xcom_calloc((size_t)1, sizeof(reply_data)));
5764 85 data->rt = leader_info;
5765 85 data->reply_data_u.leaders.max_nr_leaders = site->max_active_leaders;
5766
2/2
✓ Branch 0 taken 32 times.
✓ Branch 1 taken 53 times.
 85 if (leaders_set_by_client(site)) {
5767 data->reply_data_u.leaders.preferred_leaders =
5768 32 clone_leader_array(site->leaders);
5769 }
5770 85 active_leaders(site, &data->reply_data_u.leaders.actual_leaders);
5771 85 return data;
5772 } else {
5773 ✗ return nullptr;
5774 }
5775 }
5776
5777 85 void dispatch_get_leaders(site_def *site, pax_msg *p, linkage *reply_queue) {
5778
2/4
✓ Branch 0 taken 85 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 85 times.
✗ Branch 3 not taken.
 85 CREATE_REPLY(p);
5779 IFDBG(D_NONE, FN; STRLIT("Got get_leaders from client"); SYCEXP(p->synode););
5780 85 reply->op = xcom_client_reply;
5781
1/2
✓ Branch 0 taken 85 times.
✗ Branch 1 not taken.
 85 reply->rd = new_leader_info(site);
5782 85 reply->cli_err = reply->rd ? REQUEST_OK : REQUEST_FAIL;
5783
4/10
✓ Branch 0 taken 85 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 85 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 85 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 85 times.
✗ Branch 9 not taken.
 85 SEND_REPLY;
5784 85 }
5785
5786 /*
5787 * Log the result of the get_synode_app_data command.
5788 */
5789 ✗ static void log_get_synode_app_data_failure(
5790 xcom_get_synode_app_data_result error_code) {
5791 ✗ switch (error_code) {
5792 ✗ case XCOM_GET_SYNODE_APP_DATA_OK:
5793 ✗ break;
5794 ✗ case XCOM_GET_SYNODE_APP_DATA_ERROR:
5795 ✗ G_DEBUG("Could not reply successfully to request for synode data.");
5796 ✗ break;
5797 ✗ case XCOM_GET_SYNODE_APP_DATA_NOT_CACHED:
5798 ✗ G_DEBUG(
5799 "Could not reply successfully to request for synode data because "
5800 "some of the requested synodes are no longer cached.");
5801 ✗ break;
5802 ✗ case XCOM_GET_SYNODE_APP_DATA_NOT_DECIDED:
5803 ✗ G_DEBUG(
5804 "Could not reply successfully to request for synode data because "
5805 "some of the requested synodes are still undecided.");
5806 ✗ break;
5807 ✗ case XCOM_GET_SYNODE_APP_DATA_NO_MEMORY:
5808 ✗ G_DEBUG(
5809 "Could not reply successfully to request for synode data because "
5810 "memory could not be allocated.");
5811 ✗ break;
5812 }
5813 }
5814
5815 ✗ void dispatch_get_synode_app_data(site_def const *site, pax_msg *p,
5816 linkage *reply_queue) {
5817 IFDBG(D_NONE, FN; STRLIT("Got get_synode_app_data from client");
5818 SYCEXP(p->synode););
5819
5820 {
5821 ✗ CREATE_REPLY(p);
5822 ✗ reply->op = xcom_client_reply;
5823
5824 {
5825 xcom_get_synode_app_data_result error_code;
5826 ✗ error_code = xcom_get_synode_app_data(&p->a->body.app_u_u.synodes,
5827 ✗ &reply->requested_synode_app_data);
5828 ✗ switch (error_code) {
5829 ✗ case XCOM_GET_SYNODE_APP_DATA_OK:
5830 ✗ reply->cli_err = REQUEST_OK;
5831 ✗ break;
5832 ✗ case XCOM_GET_SYNODE_APP_DATA_NOT_CACHED:
5833 case XCOM_GET_SYNODE_APP_DATA_NOT_DECIDED:
5834 case XCOM_GET_SYNODE_APP_DATA_NO_MEMORY:
5835 case XCOM_GET_SYNODE_APP_DATA_ERROR:
5836 ✗ reply->cli_err = REQUEST_FAIL;
5837 ✗ log_get_synode_app_data_failure(error_code);
5838 ✗ break;
5839 }
5840
5841 ✗ SEND_REPLY;
5842 }
5843 }
5844 }
5845
5846 static int can_send_snapshot();
5847
5848 81186 static void process_client_msg(site_def const *site, pax_msg *p,
5849 linkage *reply_queue) {
5850 81186 clicnt++;
5851
2/4
✓ Branch 0 taken 81186 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 81186 times.
 81186 if (p->a && (p->a->body.c_t == exit_type)) {
5852 /* purecov: begin deadcode */
5853 IFDBG(D_NONE, FN; STRLIT("Got exit from client"); SYCEXP(p->synode););
5854 ✗ bury_site(get_group_id(get_site_def()));
5855 ADD_DBG(D_FSM, add_event(EVENT_DUMP_PAD, string_arg("terminating"));)
5856 ✗ terminate_and_exit();
5857 ✗ return;
5858 /* purecov: end */
5859 }
5860
5861
2/4
✓ Branch 0 taken 81186 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 81186 times.
 81186 if (p->a && (p->a->body.c_t == reset_type)) {
5862 /* purecov: begin deadcode */
5863 IFDBG(D_NONE, FN; STRLIT("Got reset from client"); SYCEXP(p->synode););
5864 ✗ bury_site(get_group_id(get_site_def()));
5865 ADD_DBG(D_FSM, add_event(EVENT_DUMP_PAD, string_arg("terminating"));)
5866 ✗ XCOM_FSM(x_fsm_terminate, int_arg(0));
5867 ✗ return;
5868 /* purecov: end */
5869 }
5870
2/4
✓ Branch 0 taken 81186 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 81186 times.
 81186 if (p->a && (p->a->body.c_t == remove_reset_type)) {
5871 /* purecov: begin deadcode */
5872 IFDBG(D_NONE, FN; STRLIT("Got remove_reset from client");
5873 SYCEXP(p->synode););
5874 ADD_DBG(D_FSM, add_event(EVENT_DUMP_PAD, string_arg("terminating"));)
5875 ✗ XCOM_FSM(x_fsm_terminate, int_arg(0));
5876 ✗ return;
5877 /* purecov: end */
5878 }
5879
2/4
✓ Branch 0 taken 81186 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 81186 times.
 81186 if (p->a && (p->a->body.c_t == enable_arbitrator)) {
5880 ✗ CREATE_REPLY(p);
5881 IFDBG(D_NONE, FN; STRLIT("Got enable_arbitrator from client");
5882 SYCEXP(p->synode););
5883 ✗ ARBITRATOR_HACK = 1;
5884 ✗ reply->op = xcom_client_reply;
5885 ✗ reply->cli_err = REQUEST_OK;
5886 ✗ SEND_REPLY;
5887 ✗ return;
5888 }
5889
2/4
✓ Branch 0 taken 81186 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 81186 times.
 81186 if (p->a && (p->a->body.c_t == disable_arbitrator)) {
5890 ✗ CREATE_REPLY(p);
5891 IFDBG(D_NONE, FN; STRLIT("Got disable_arbitrator from client");
5892 SYCEXP(p->synode););
5893 ✗ ARBITRATOR_HACK = 0;
5894 ✗ reply->op = xcom_client_reply;
5895 ✗ reply->cli_err = REQUEST_OK;
5896 ✗ SEND_REPLY;
5897 ✗ return;
5898 }
5899
3/4
✓ Branch 0 taken 81186 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 3 times.
✓ Branch 3 taken 81183 times.
 81186 if (p->a && (p->a->body.c_t == set_cache_limit)) {
5900
2/4
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 3 times.
✗ Branch 3 not taken.
 3 CREATE_REPLY(p);
5901 IFDBG(D_NONE, FN; STRLIT("Got set_cache_limit from client");
5902 SYCEXP(p->synode););
5903
1/2
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
 3 if (the_app_xcom_cfg) {
5904
1/2
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
 3 set_max_cache_size(p->a->body.app_u_u.cache_limit);
5905 3 reply->cli_err = REQUEST_OK;
5906 } else {
5907 ✗ reply->cli_err = REQUEST_FAIL;
5908 }
5909 3 reply->op = xcom_client_reply;
5910
4/10
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 3 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 3 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 3 times.
✗ Branch 9 not taken.
 3 SEND_REPLY;
5911 3 return;
5912 }
5913
2/4
✓ Branch 0 taken 81183 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 81183 times.
 81183 if (p->a && (p->a->body.c_t == x_terminate_and_exit)) {
5914 /* purecov: begin deadcode */
5915 ✗ CREATE_REPLY(p);
5916 IFDBG(D_NONE, FN; STRLIT("Got terminate_and_exit from client");
5917 SYCEXP(p->synode););
5918 ✗ reply->op = xcom_client_reply;
5919 ✗ reply->cli_err = REQUEST_OK;
5920 ✗ SEND_REPLY;
5921 /*
5922 The function frees sites which is used by SEND_REPLY,
5923 so it should be called after SEND_REPLY.
5924 */
5925 IFDBG(D_NONE, FN; STRLIT("terminate_and_exit"));
5926 ADD_DBG(D_FSM, add_event(EVENT_DUMP_PAD, string_arg("terminating"));)
5927 ✗ terminate_and_exit();
5928 ✗ return;
5929 /* purecov: end */
5930 }
5931
3/4
✓ Branch 0 taken 81183 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 99 times.
✓ Branch 3 taken 81084 times.
 81183 if (p->a && (p->a->body.c_t == get_event_horizon_type)) {
5932 99 dispatch_get_event_horizon(get_site_def(), p, reply_queue);
5933 99 return;
5934 }
5935
2/4
✓ Branch 0 taken 81084 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 81084 times.
 81084 if (p->a && (p->a->body.c_t == get_synode_app_data_type)) {
5936 ✗ dispatch_get_synode_app_data(get_site_def(), p, reply_queue);
5937 ✗ return;
5938 }
5939
3/4
✓ Branch 0 taken 81084 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 85 times.
✓ Branch 3 taken 80999 times.
 81084 if (p->a && (p->a->body.c_t == get_leaders_type)) {
5940 85 dispatch_get_leaders(get_site_def_rw(), p, reply_queue);
5941 85 return;
5942 }
5943
1/2
✓ Branch 0 taken 80999 times.
✗ Branch 1 not taken.
 80999 if (p->a &&
5944
4/4
✓ Branch 0 taken 79168 times.
✓ Branch 1 taken 1831 times.
✓ Branch 2 taken 76858 times.
✓ Branch 3 taken 2310 times.
 80999 (p->a->body.c_t == add_node_type || p->a->body.c_t == remove_node_type ||
5945
2/2
✓ Branch 0 taken 76830 times.
✓ Branch 1 taken 28 times.
 76858 p->a->body.c_t == force_config_type ||
5946
2/2
✓ Branch 0 taken 76821 times.
✓ Branch 1 taken 9 times.
 76830 p->a->body.c_t == set_event_horizon_type ||
5947
1/2
✓ Branch 0 taken 76821 times.
✗ Branch 1 not taken.
 76821 p->a->body.c_t == set_max_leaders ||
5948
2/2
✓ Branch 0 taken 39 times.
✓ Branch 1 taken 76782 times.
 76821 p->a->body.c_t == set_leaders_type)) {
5949 client_reply_code cli_err;
5950
2/4
✓ Branch 0 taken 4217 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 4217 times.
✗ Branch 3 not taken.
 4217 CREATE_REPLY(p);
5951 4217 reply->op = xcom_client_reply;
5952
1/2
✓ Branch 0 taken 4217 times.
✗ Branch 1 not taken.
 4217 reply->cli_err = cli_err = can_execute_cfgchange(p);
5953
4/10
✓ Branch 0 taken 4217 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 4217 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 4217 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 4217 times.
✗ Branch 9 not taken.
 4217 SEND_REPLY;
5954
2/2
✓ Branch 0 taken 552 times.
✓ Branch 1 taken 3665 times.
 4217 if (cli_err != REQUEST_OK) {
5955 552 return;
5956 }
5957 }
5958
3/4
✓ Branch 0 taken 80447 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 848 times.
✓ Branch 3 taken 79599 times.
 80447 if (p->a && p->a->body.c_t == unified_boot_type) {
5959 IFDBG(D_NONE, FN; STRLIT("Got unified_boot from client");
5960 SYCEXP(p->synode););
5961 IFDBG(D_NONE, FN; COPY_AND_FREE_GOUT(dbg_list(&p->a->body.app_u_u.nodes)););
5962 IFDBG(D_NONE, STRLIT("handle_client_msg "); NDBG(p->a->group_id, x));
5963
4/4
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 847 times.
✓ Branch 2 taken 1 times.
✓ Branch 3 taken 847 times.
 848 XCOM_FSM(x_fsm_net_boot, void_arg(p->a));
5964 }
5965
3/4
✓ Branch 0 taken 80447 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1329 times.
✓ Branch 3 taken 79118 times.
 80447 if (p->a && p->a->body.c_t == add_node_type) {
5966 IFDBG(D_NONE, FN; STRLIT("Got add_node from client"); SYCEXP(p->synode););
5967 IFDBG(D_NONE, FN; COPY_AND_FREE_GOUT(dbg_list(&p->a->body.app_u_u.nodes)););
5968 IFDBG(D_NONE, STRLIT("handle_client_msg "); NDBG(p->a->group_id, x));
5969
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1329 times.
 1329 assert(get_site_def());
5970 }
5971
3/4
✓ Branch 0 taken 80447 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2261 times.
✓ Branch 3 taken 78186 times.
 80447 if (p->a && p->a->body.c_t == remove_node_type) {
5972 IFDBG(D_NONE, FN; STRLIT("Got remove_node from client");
5973 SYCEXP(p->synode););
5974 IFDBG(D_NONE, FN; COPY_AND_FREE_GOUT(dbg_list(&p->a->body.app_u_u.nodes)););
5975 IFDBG(D_NONE, STRLIT("handle_client_msg "); NDBG(p->a->group_id, x));
5976
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2261 times.
 2261 assert(get_site_def());
5977 }
5978
3/4
✓ Branch 0 taken 80447 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 9 times.
✓ Branch 3 taken 80438 times.
 80447 if (p->a && p->a->body.c_t == set_event_horizon_type) {
5979 IFDBG(D_NONE, FN; STRLIT("Got set_event_horizon from client");
5980 SYCEXP(p->synode););
5981 IFDBG(D_NONE, FN; NDBG(p->a->body.app_u_u.event_horizon, u));
5982 IFDBG(D_NONE, STRLIT("handle_client_msg "); NDBG(p->a->group_id, x));
5983
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 9 times.
 9 assert(get_site_def());
5984 }
5985
3/4
✓ Branch 0 taken 80447 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 27 times.
✓ Branch 3 taken 80420 times.
 80447 if (p->a && p->a->body.c_t == force_config_type) {
5986 IFDBG(D_NONE, FN; STRLIT("Got new force config from client");
5987 SYCEXP(p->synode););
5988 IFDBG(D_NONE, FN; COPY_AND_FREE_GOUT(dbg_list(&p->a->body.app_u_u.nodes)););
5989 IFDBG(D_NONE, STRLIT("handle_client_msg "); NDBG(p->a->group_id, x));
5990
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 27 times.
 27 assert(get_site_def());
5991
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 27 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 27 times.
 27 XCOM_FSM(x_fsm_force_config, void_arg(p->a));
5992 }
5993
2/4
✓ Branch 0 taken 80447 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 80447 times.
 80447 if (p->a && p->a->body.c_t == set_max_leaders) {
5994 IFDBG(D_NONE, FN; STRLIT("Got set_max_leaders from client");
5995 SYCEXP(p->synode););
5996 IFDBG(D_NONE, FN; NDBG(p->a->body.app_u_u.max_leaders, u));
5997 IFDBG(D_NONE, STRLIT("handle_client_msg "); NDBG(p->a->group_id, x));
5998 ✗ assert(get_site_def());
5999 }
6000
3/4
✓ Branch 0 taken 80447 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 39 times.
✓ Branch 3 taken 80408 times.
 80447 if (p->a && p->a->body.c_t == set_leaders_type) {
6001 IFDBG(D_NONE, FN; STRLIT("Got set_leaders_type from client");
6002 SYCEXP(p->synode););
6003 IFDBG(D_NONE, STRLIT("handle_client_msg "); NDBG(p->a->group_id, x));
6004
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 39 times.
 39 assert(get_site_def());
6005 }
6006 80447 handle_client_msg(p);
6007 }
6008
6009 46153 static void process_prepare_op(site_def const *site, pax_msg *p,
6010 linkage *reply_queue) {
6011 46153 pax_machine *pm = get_cache(p->synode);
6012
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 46153 times.
 46153 assert(pm);
6013
2/2
✓ Branch 0 taken 1692 times.
✓ Branch 1 taken 44461 times.
 46153 if (p->force_delivery) pm->force_delivery = 1;
6014 IFDBG(D_NONE, FN; dbg_pax_msg(p));
6015
6016 /*
6017 We can only be a productive Paxos Acceptor if we have been booted, i.e.
6018 added to the group and received an up-to-date snapshot from some member.
6019
6020 We do not allow non-booted members to participate in Paxos because they
6021 might be a reincarnation of a member that crashed and was then brought up
6022 without having gone through the remove+add node path.
6023 Since the pre-crash incarnation may have accepted a value for a given
6024 synod but the post-crash incarnation has forgotten that fact, the
6025 post-crash incarnation will fail to propagate the previously accepted
6026 value to a higher ballot. Since majorities can overlap on a single node,
6027 if the overlap node is the post-crash incarnation which has forgotten
6028 about the previously accepted value, the higher ballot proposer may get
6029 a different value accepted, leading to conflicting values to be accepted
6030 for different proposers, which is a violation of the safety requirements
6031 of the Paxos protocol.
6032 */
6033
2/2
✓ Branch 0 taken 34639 times.
✓ Branch 1 taken 11514 times.
 46153 if (ALWAYS_HANDLE_CONSENSUS || client_boot_done) {
6034 34639 paxos_fsm(pm, site, paxos_prepare, p);
6035 34639 handle_prepare(site, pm, reply_queue, p);
6036 }
6037 46153 }
6038
6039 32110 static inline int abort_processing(pax_msg *p) {
6040 /* Ensure that forced message can be processed */
6041
4/6
✓ Branch 0 taken 30418 times.
✓ Branch 1 taken 1692 times.
✓ Branch 2 taken 30418 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 32110 times.
 32110 return (!p->force_delivery && too_far(p->synode)) || !is_cached(p->synode);
6042 }
6043
6044 32110 static void process_ack_prepare_op(site_def const *site, pax_msg *p,
6045 linkage *reply_queue) {
6046 (void)reply_queue;
6047
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 32110 times.
 32110 if (abort_processing(p))
6048 ✗ return;
6049 else {
6050 32110 pax_machine *pm = get_cache(p->synode);
6051
2/2
✓ Branch 0 taken 1692 times.
✓ Branch 1 taken 30418 times.
 32110 if (p->force_delivery) pm->force_delivery = 1;
6052
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 32110 times.
 32110 if (!pm->proposer.msg) return;
6053
2/4
✓ Branch 0 taken 32110 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 32110 times.
✗ Branch 3 not taken.
 32110 assert(pm && pm->proposer.msg);
6054 32110 handle_ack_prepare(site, pm, p);
6055 32110 paxos_fsm(pm, site, paxos_ack_prepare, p);
6056 }
6057 }
6058
6059 187982 static void process_accept_op(site_def const *site, pax_msg *p,
6060 linkage *reply_queue) {
6061 187982 pax_machine *pm = get_cache(p->synode);
6062
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 187982 times.
 187982 assert(pm);
6063
2/2
✓ Branch 0 taken 82 times.
✓ Branch 1 taken 187900 times.
 187982 if (p->force_delivery) pm->force_delivery = 1;
6064 IFDBG(D_NONE, FN; dbg_pax_msg(p));
6065
6066 /*
6067 We can only be a productive Paxos Acceptor if we have been booted, i.e.
6068 added to the group and received an up-to-date snapshot from some member.
6069
6070 We do not allow non-booted members to participate in Paxos because they
6071 might be a reincarnation of a member that crashed and was then brought up
6072 without having gone through the remove+add node path.
6073 Since the pre-crash incarnation may have accepted a value for a given
6074 synod but the post-crash incarnation has forgotten that fact, the
6075 post-crash incarnation will fail to propagate the previously accepted
6076 value to a higher ballot. Since majorities can overlap on a single node,
6077 if the overlap node is the post-crash incarnation which has forgotten
6078 about the previously accepted value, the higher ballot proposer may get
6079 a different value accepted, leading to conflicting values to be accepted
6080 for different proposers, which is a violation of the safety requirements
6081 of the Paxos protocol.
6082 */
6083
2/2
✓ Branch 0 taken 184362 times.
✓ Branch 1 taken 3620 times.
 187982 if (ALWAYS_HANDLE_CONSENSUS || client_boot_done) {
6084 184362 handle_alive(site, reply_queue, p);
6085
6086 184362 paxos_fsm(pm, site, paxos_accept, p);
6087 184362 handle_accept(site, pm, reply_queue, p);
6088 }
6089 187982 }
6090
6091 179720 static void process_ack_accept_op(site_def const *site, pax_msg *p,
6092 linkage *reply_queue) {
6093 (void)reply_queue;
6094
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 179720 times.
 179720 if (too_far(p->synode))
6095 ✗ return;
6096 else {
6097 179720 pax_machine *pm = get_cache(p->synode);
6098
2/2
✓ Branch 0 taken 82 times.
✓ Branch 1 taken 179638 times.
 179720 if (p->force_delivery) pm->force_delivery = 1;
6099
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 179720 times.
 179720 if (!pm->proposer.msg) return;
6100
2/4
✓ Branch 0 taken 179720 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 179720 times.
✗ Branch 3 not taken.
 179720 assert(pm && pm->proposer.msg);
6101 179720 handle_ack_accept(site, pm, p);
6102 179720 paxos_fsm(pm, site, paxos_ack_accept, p);
6103 }
6104 }
6105
6106 234511 static void process_learn_op(site_def const *site, pax_msg *p,
6107 linkage *reply_queue) {
6108 234511 pax_machine *pm = get_cache(p->synode);
6109
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 234511 times.
 234511 assert(pm);
6110 (void)reply_queue;
6111
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 234511 times.
 234511 if (p->force_delivery) pm->force_delivery = 1;
6112 234511 update_max_synode(p);
6113 234511 paxos_fsm(pm, site, paxos_learn, p);
6114 234511 handle_learn(site, pm, p);
6115 234511 }
6116
6117 45302 static void process_recover_learn_op(site_def const *site, pax_msg *p,
6118 linkage *reply_queue) {
6119 45302 pax_machine *pm = get_cache(p->synode);
6120
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 45302 times.
 45302 assert(pm);
6121 (void)reply_queue;
6122 IFDBG(D_NONE, FN; STRLIT("recover_learn_op receive "); SYCEXP(p->synode));
6123
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 45302 times.
 45302 if (p->force_delivery) pm->force_delivery = 1;
6124 45302 update_max_synode(p);
6125 {
6126 IFDBG(D_NONE, FN; STRLIT("recover_learn_op learn "); SYCEXP(p->synode));
6127 45302 p->op = learn_op;
6128 45302 paxos_fsm(pm, site, paxos_learn, p);
6129 45302 handle_learn(site, pm, p);
6130 }
6131 45302 }
6132
6133 249014 static void process_skip_op(site_def const *site, pax_msg *p,
6134 linkage *reply_queue) {
6135 (void)reply_queue;
6136 249014 pax_machine *pm = get_cache(p->synode);
6137
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 249014 times.
 249014 assert(pm);
6138
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 249014 times.
 249014 if (p->force_delivery) pm->force_delivery = 1;
6139 249014 paxos_fsm(pm, site, paxos_learn, p);
6140 249014 handle_skip(site, pm, p);
6141 249014 }
6142
6143 98353 static void process_i_am_alive_op(site_def const *site, pax_msg *p,
6144 linkage *reply_queue) {
6145 /* Update max_synode, but use only p->max_synode, ignore p->synode */
6146
1/2
✓ Branch 0 taken 98353 times.
✗ Branch 1 not taken.
 98353 if (!is_dead_site(p->group_id)) {
6147
4/4
✓ Branch 0 taken 98088 times.
✓ Branch 1 taken 265 times.
✓ Branch 2 taken 2057 times.
✓ Branch 3 taken 96296 times.
 196441 if (max_synode.group_id == p->synode.group_id &&
6148
2/2
✓ Branch 0 taken 2057 times.
✓ Branch 1 taken 96031 times.
 98088 synode_gt(p->max_synode, max_synode)) {
6149 2057 set_max_synode(p->max_synode);
6150 }
6151 }
6152 98353 handle_alive(site, reply_queue, p);
6153 98353 }
6154
6155 5568 static void process_are_you_alive_op(site_def const *site, pax_msg *p,
6156 linkage *reply_queue) {
6157 5568 handle_alive(site, reply_queue, p);
6158 5568 }
6159
6160 ✗ static void process_need_boot_op(site_def const *site, pax_msg *p,
6161 linkage *reply_queue) {
6162 /* purecov: begin deadcode */
6163 /* Only in run state. Test state and do it here because we need to use
6164 * reply queue */
6165 ✗ if (can_send_snapshot() &&
6166 ✗ !synode_eq(get_site_def()->boot_key, null_synode)) {
6167 ✗ handle_boot(site, reply_queue, p);
6168 }
6169 /* Wake senders waiting to connect, since new node has appeared */
6170 ✗ wakeup_sender();
6171 /* purecov: end */
6172 }
6173
6174 ✗ static void process_die_op(site_def const *site, pax_msg *p,
6175 linkage *reply_queue) {
6176 (void)reply_queue;
6177 /* assert("die horribly" == "need coredump"); */
6178 {
6179 ✗ GET_GOUT;
6180 ✗ FN;
6181 ✗ STRLIT("die_op ");
6182 ✗ SYCEXP(executed_msg);
6183 ✗ SYCEXP(delivered_msg);
6184 ✗ SYCEXP(p->synode);
6185 ✗ SYCEXP(p->delivered_msg);
6186 ✗ SYCEXP(p->max_synode);
6187 ✗ PRINT_GOUT;
6188 ✗ FREE_GOUT;
6189 }
6190 /*
6191 If the message with the number in the incoming die_op message
6192 already has been executed (delivered), then it means that we
6193 actually got consensus on it, since otherwise we would not have
6194 delivered it.Such a situation could arise if one of the nodes has
6195 expelled the message from its cache, but others have not. So when
6196 sending out a request, we might get two different answers, one
6197 indicating that we are too far behind and should restart, and
6198 another with the actual consensus value. If the value arrives
6199 first, we will deliver it, and then the die_op may arrive later.
6200 But it this case it does not matter, since we got what we needed
6201 anyway. It is only a partial guard against exiting without really
6202 needing it of course, since the die_op may arrive first, and we
6203 do not wait for a die_op from all the other nodes. We could do
6204 that with some extra housekeeping in the pax_machine (a new bit
6205 vector), but I am not convinced that it is worth the effort.
6206 */
6207 ✗ if (!synode_lt(p->synode, executed_msg)) {
6208 ADD_DBG(D_FSM, add_event(EVENT_DUMP_PAD, string_arg("terminating"));)
6209 ✗ g_critical("Node %u is unable to get message {%x %" PRIu64
6210 " %u}, since the group is too far "
6211 "ahead. Node will now exit.",
6212 get_nodeno(site), SY_MEM(p->synode));
6213 ✗ terminate_and_exit();
6214 }
6215 }
6216
6217 345299 static void process_read_op(site_def const *site, pax_msg *p,
6218 linkage *reply_queue) {
6219 345299 pax_machine *pm = get_cache(p->synode);
6220
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 345299 times.
 345299 assert(pm);
6221
6222 345299 handle_read(site, pm, reply_queue, p);
6223 345299 }
6224
6225 1367 static void process_gcs_snapshot_op(site_def const *site, pax_msg *p,
6226 linkage *reply_queue) {
6227 (void)site;
6228 (void)reply_queue;
6229 /* Avoid duplicate snapshots and snapshots from zombies */
6230 IFDBG(D_BASE, FN; SYCEXP(executed_msg););
6231 IFDBG(D_BASE, FN; SYCEXP(start_config););
6232
3/6
✓ Branch 0 taken 1367 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1367 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1367 times.
✗ Branch 5 not taken.
 2734 if (!synode_eq(start_config, get_highest_boot_key(p->gcs_snap)) &&
6233 1367 !is_dead_site(p->group_id)) {
6234 1367 update_max_synode(p);
6235 /* For incoming messages, note delivery of snapshot from sender node */
6236 1367 note_snapshot(p->from);
6237
3/4
✗ Branch 0 not taken.
✓ Branch 1 taken 1367 times.
✓ Branch 2 taken 1 times.
✓ Branch 3 taken 1366 times.
 1367 XCOM_FSM(x_fsm_snapshot, void_arg(p->gcs_snap));
6238 }
6239 1367 }
6240
6241 184994 static void process_tiny_learn_op(site_def const *site, pax_msg *p,
6242 linkage *reply_queue) {
6243
2/2
✓ Branch 0 taken 23302 times.
✓ Branch 1 taken 161692 times.
 184994 if (p->msg_type == no_op) {
6244 23302 process_learn_op(site, p, reply_queue);
6245 } else {
6246 161692 pax_machine *pm = get_cache(p->synode);
6247
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 161692 times.
 161692 assert(pm);
6248
2/2
✓ Branch 0 taken 82 times.
✓ Branch 1 taken 161610 times.
 161692 if (p->force_delivery) pm->force_delivery = 1;
6249 161692 handle_tiny_learn(site, pm, p);
6250 }
6251 184994 }
6252
6253 /* If this node is leader, grant a synode number for use by secondary.
6254 Send reply as synode_allocated. */
6255 420 static void process_synode_request(site_def const *site, pax_msg *p,
6256 linkage *reply_queue) {
6257 (void)site;
6258
6259 /* Find a free slot */
6260
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 420 times.
 420 assert(!synode_eq(current_message, null_synode));
6261 IFDBG(D_CONS, FN; SYCEXP(executed_msg); SYCEXP(current_message));
6262 420 site_def *tmp_site = find_site_def_rw(current_message);
6263 /* See if we can do anything with this message */
6264
6/8
✓ Branch 0 taken 420 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 420 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 419 times.
✓ Branch 5 taken 1 times.
✓ Branch 6 taken 419 times.
✓ Branch 7 taken 1 times.
 420 if (tmp_site && get_nodeno(tmp_site) != VOID_NODE_NO && is_leader(tmp_site)) {
6265 /* Send reply with msgno */
6266
1/2
✓ Branch 0 taken 419 times.
✗ Branch 1 not taken.
 419 synode_no msgno = local_synode_allocator(current_message);
6267 /* Ensure that reply is sane. Note that we only allocate `msgno` *if* next
6268 synod is still within the event horizon. This effectively means that
6269 the leader always reserves at least one synod to himself, the last
6270 synod of the event horizon. The point is to ensure that the leader does
6271 not allocate all the possible synods to a non-leader that then doesn't
6272 act on them, e.g. by crashing. The last synod of the event horizon will
6273 always be up for grabs either by the leader, or if the leader is
6274 suspected, by some non-leader that will self-allocate that synod to
6275 himself. */
6276
6/8
✓ Branch 0 taken 419 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 419 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 398 times.
✓ Branch 5 taken 21 times.
✓ Branch 6 taken 398 times.
✓ Branch 7 taken 21 times.
 817 if (!(too_far(incr_msgno(msgno)) ||
6277
3/6
✓ Branch 0 taken 398 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 398 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 398 times.
✗ Branch 5 not taken.
 398 ignore_message(msgno, find_site_def_rw(msgno),
6278 "process_synode_request"))) {
6279 // We will grab this number, advance current_message
6280
2/4
✓ Branch 0 taken 398 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 398 times.
✗ Branch 3 not taken.
 398 set_current_message(incr_synode(msgno));
6281 IFDBG(D_CONS, FN; STRLIT("sending reply "); SYCEXP(executed_msg);
6282 SYCEXP(current_message); SYCEXP(msgno));
6283
2/4
✓ Branch 0 taken 398 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 398 times.
✗ Branch 3 not taken.
 398 CREATE_REPLY(p);
6284 398 reply->synode = msgno;
6285 398 reply->op = synode_allocated;
6286 IFDBG(D_CONS, FN; SYCEXP(msgno));
6287
4/10
✓ Branch 0 taken 398 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 398 times.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 398 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 398 times.
✗ Branch 9 not taken.
 398 SEND_REPLY;
6288 } else {
6289 IFDBG(D_CONS, FN; STRLIT("not sending reply "); SYCEXP(executed_msg);
6290 SYCEXP(msgno));
6291 }
6292 } else {
6293 IFDBG(D_CONS, FN; STRLIT("not leader ");
6294 if (tmp_site) SYCEXP(tmp_site->start));
6295 }
6296 420 }
6297
6298 /* If this node is secondary, add synode to set of available synodes */
6299 402 static void process_synode_allocated(site_def const *site, pax_msg *p,
6300 linkage *reply_queue) {
6301 (void)site;
6302 (void)p;
6303 (void)reply_queue;
6304
6305 IFDBG(D_BASE, FN; SYCEXP(p->synode));
6306 402 synode_number_pool.put(p->synode, synode_allocation_type::remote);
6307 402 }
6308
6309 static msg_handler dispatch_table[LAST_OP] = {process_client_msg,
6310 nullptr,
6311 process_prepare_op,
6312 process_ack_prepare_op,
6313 process_ack_prepare_op,
6314 process_accept_op,
6315 process_ack_accept_op,
6316 process_learn_op,
6317 process_recover_learn_op,
6318 nullptr,
6319 nullptr,
6320 nullptr,
6321 nullptr,
6322 nullptr,
6323 process_skip_op,
6324 process_i_am_alive_op,
6325 process_are_you_alive_op,
6326 process_need_boot_op,
6327 nullptr,
6328 process_die_op,
6329 process_read_op,
6330 process_gcs_snapshot_op,
6331 nullptr,
6332 process_tiny_learn_op,
6333 process_synode_request,
6334 process_synode_allocated};
6335
6336 58 static msg_handler *clone_dispatch_table(msg_handler const *proto) {
6337 msg_handler *clone =
6338 58 static_cast<msg_handler *>(xcom_calloc(1, sizeof(dispatch_table)));
6339
1/2
✓ Branch 0 taken 58 times.
✗ Branch 1 not taken.
 58 if (clone)
6340 58 memcpy(clone, proto, sizeof(dispatch_table));
6341 else
6342 ✗ oom_abort = 1;
6343 58 return clone;
6344 }
6345
6346 25 static msg_handler *primary_dispatch_table() {
6347 25 msg_handler *clone = clone_dispatch_table(dispatch_table);
6348 25 return clone;
6349 }
6350
6351 33 static msg_handler *secondary_dispatch_table() {
6352 33 msg_handler *clone = clone_dispatch_table(dispatch_table);
6353
1/2
✓ Branch 0 taken 33 times.
✗ Branch 1 not taken.
 33 if (clone) {
6354 33 clone[synode_request] = nullptr;
6355 }
6356 33 return clone;
6357 }
6358
6359 1669079 pax_msg *dispatch_op(site_def const *site, pax_msg *p, linkage *reply_queue) {
6360 1669079 site_def *dsite = find_site_def_rw(p->synode);
6361
6362
8/8
✓ Branch 0 taken 1574286 times.
✓ Branch 1 taken 94793 times.
✓ Branch 2 taken 1527009 times.
✓ Branch 3 taken 47277 times.
✓ Branch 4 taken 1318749 times.
✓ Branch 5 taken 208260 times.
✓ Branch 6 taken 1318749 times.
✓ Branch 7 taken 350330 times.
 1669079 if (dsite && p->op != client_msg && is_server_connected(dsite, p->from)) {
6363 /* Wake up the detector task if this node was previously marked as
6364 * potentially failed. */
6365
2/2
✓ Branch 0 taken 46011 times.
✓ Branch 1 taken 1272738 times.
 1318749 if (!note_detected(dsite, p->from)) task_wakeup(&detector_wait);
6366 1318749 update_delivered(dsite, p->from, p->delivered_msg);
6367 }
6368
6369 IFDBG(D_BASE, FN; STRLIT("incoming message ");
6370 COPY_AND_FREE_GOUT(dbg_pax_msg(p)););
6371 ADD_DBG(D_DISPATCH, add_synode_event(p->synode);
6372 add_event(EVENT_DUMP_PAD, string_arg("p->from"));
6373 add_event(EVENT_DUMP_PAD, uint_arg(p->from));
6374 add_event(EVENT_DUMP_PAD, string_arg("too_far(p->synode)"));
6375 add_event(EVENT_DUMP_PAD, int_arg(too_far(p->synode)));
6376 add_event(EVENT_DUMP_PAD, string_arg(pax_op_to_str(p->op))););
6377
6378
2/4
✓ Branch 0 taken 1669079 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1669079 times.
✗ Branch 3 not taken.
 1669079 if (p->op >= 0 && p->op < LAST_OP) {
6379
4/4
✓ Branch 0 taken 1527009 times.
✓ Branch 1 taken 142070 times.
✓ Branch 2 taken 12073 times.
✓ Branch 3 taken 1514936 times.
 1669079 if (site && site->dispatch_table) { /* Use site-specific dispatch if any */
6380
1/2
✓ Branch 0 taken 12073 times.
✗ Branch 1 not taken.
 12073 if (site->dispatch_table[p->op])
6381 12073 site->dispatch_table[p->op](site, p, reply_queue);
6382 } else {
6383
1/2
✓ Branch 0 taken 1657006 times.
✗ Branch 1 not taken.
 1657006 if (dispatch_table[p->op]) dispatch_table[p->op](site, p, reply_queue);
6384 }
6385 } else {
6386 ✗ G_WARNING("No possible handler for message %d %s", p->op,
6387 pax_op_to_str(p->op));
6388 }
6389
6390
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1669079 times.
 1669079 if (oom_abort) {
6391 ✗ g_critical("Node %u has run out of memory and will now exit.",
6392 get_nodeno(site));
6393 ✗ terminate_and_exit();
6394 }
6395 1669079 return (p);
6396 }
6397
6398 /* Acceptor-learner task */
6399 #define SERIALIZE_REPLY(msg) \
6400 msg->to = ep->p->from; \
6401 msg->from = ep->p->to; \
6402 msg->delivered_msg = get_delivered_msg(); \
6403 msg->max_synode = get_max_synode(); \
6404 serialize_msg(msg, ep->rfd->x_proto, &ep->buflen, &ep->buf);
6405
6406 #define WRITE_REPLY \
6407 if (ep->buflen) { \
6408 int64_t sent; \
6409 IFDBG(D_TRANSPORT, FN; STRLIT("task_write "); NDBG(ep->rfd.fd, d); \
6410 NDBG(ep->buflen, u)); \
6411 TASK_CALL(task_write(ep->rfd, ep->buf, ep->buflen, &sent)); \
6412 send_count[ep->p->op]++; \
6413 send_bytes[ep->p->op] += ep->buflen; \
6414 X_FREE(ep->buf); \
6415 } \
6416 ep->buf = NULL;
6417
6418 898443 static inline void update_srv(server **target, server *srv) {
6419
2/2
✓ Branch 0 taken 625571 times.
✓ Branch 1 taken 272872 times.
 898443 if (srv) srv_ref(srv);
6420
2/2
✓ Branch 0 taken 625560 times.
✓ Branch 1 taken 272883 times.
 898443 if (*target) srv_unref(*target);
6421 898443 *target = srv;
6422 898443 }
6423
6424 /* A message is harmless if it cannot change the outcome of a consensus round.
6425 * learn_op does change the value, but we trust that the sender has correctly
6426 * derived the value from a majority of the acceptors, so in that sense it is
6427 * harmless. */
6428 889496 static int harmless(pax_msg const *p) {
6429
2/2
✓ Branch 0 taken 1858 times.
✓ Branch 1 taken 887638 times.
 889496 if (p->synode.msgno == 0) return 1;
6430
2/2
✓ Branch 0 taken 282935 times.
✓ Branch 1 taken 604703 times.
 887638 switch (p->op) {
6431 282935 case i_am_alive_op:
6432 case are_you_alive_op:
6433 case need_boot_op:
6434 case gcs_snapshot_op:
6435 case learn_op:
6436 case recover_learn_op:
6437 case tiny_learn_op:
6438 case die_op:
6439 282935 return 1;
6440 604703 default:
6441 604703 return 0;
6442 }
6443 }
6444
6445 922052 static int wait_for_cache(pax_machine **pm, synode_no synode, double timeout) {
6446 DECL_ENV
6447 double now;
6448 922052 ENV_INIT
6449 922052 END_ENV_INIT
6450 END_ENV;
6451
6452
4/9
✓ Branch 0 taken 922052 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 922052 times.
✓ Branch 5 taken 922052 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 922052 times.
 922052 TASK_BEGIN
6453 922052 ep->now = task_now();
6454
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 922052 times.
 922052 while ((*pm = get_cache(synode)) == nullptr) {
6455 /* Wait for executor to make progress */
6456 ✗ TIMED_TASK_WAIT(&exec_wait, 0.5);
6457 ✗ if (task_now() - ep->now > timeout) break; /* Timeout, return NULL. */
6458 }
6459 922052 FINALLY
6460
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 922052 times.
 922052 TASK_END;
6461 }
6462
6463 /*
6464 Verify if we need to poll the cache before calling dispatch_op.
6465 Avoid waiting for a machine if it is not going to be used.
6466 */
6467 841583 static bool_t should_poll_cache(pax_op op) {
6468
6/8
✓ Branch 0 taken 841583 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 840216 times.
✓ Branch 3 taken 1367 times.
✓ Branch 4 taken 840216 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 1858 times.
✓ Branch 7 taken 838358 times.
 841583 if (op == die_op || op == gcs_snapshot_op || op == initial_op ||
6469 op == client_msg)
6470 3225 return FALSE;
6471 838358 return TRUE;
6472 }
6473
6474 339690 int acceptor_learner_task(task_arg arg) {
6475 DECL_ENV
6476 connection_descriptor *rfd;
6477 srv_buf *in_buf;
6478 pax_msg *p;
6479 u_int buflen;
6480 char *buf;
6481 linkage reply_queue;
6482 int errors;
6483 server *srv;
6484 site_def const *site;
6485 int behind;
6486 7867 ENV_INIT
6487 7867 END_ENV_INIT
6488 END_ENV;
6489
6490 339690 int64_t n{0};
6491 339690 pax_machine *pm{nullptr};
6492
6493
8/21
✓ Branch 0 taken 7867 times.
✓ Branch 1 taken 7867 times.
✓ Branch 2 taken 323956 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✓ Branch 11 taken 7867 times.
✗ Branch 12 not taken.
✓ Branch 13 taken 7867 times.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✓ Branch 16 taken 7867 times.
✓ Branch 17 taken 7867 times.
✗ Branch 18 not taken.
✗ Branch 19 not taken.
✓ Branch 20 taken 7867 times.
 339690 TASK_BEGIN
6494
6495 7867 ep->rfd = (connection_descriptor *)get_void_arg(arg);
6496 7867 ep->in_buf = (srv_buf *)xcom_calloc(1, sizeof(srv_buf));
6497 7867 ep->p = nullptr;
6498 7867 ep->buflen = 0;
6499 7867 ep->buf = nullptr;
6500 7867 ep->errors = 0;
6501 7867 ep->srv = nullptr;
6502 7867 ep->behind = FALSE;
6503
6504 /* We have a connection, make socket non-blocking and wait for request */
6505
1/2
✓ Branch 0 taken 7867 times.
✗ Branch 1 not taken.
 7867 unblock_fd(ep->rfd->fd);
6506
1/2
✓ Branch 0 taken 7867 times.
✗ Branch 1 not taken.
 7867 set_nodelay(ep->rfd->fd);
6507
1/2
✓ Branch 0 taken 7867 times.
✗ Branch 1 not taken.
 7867 wait_io(stack, ep->rfd->fd, 'r');
6508
3/6
✗ Branch 0 not taken.
✓ Branch 1 taken 7867 times.
✓ Branch 2 taken 7867 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 7867 times.
 15734 TASK_YIELD;
6509
6510 7867 set_connected(ep->rfd, CON_FD);
6511 7867 link_init(&ep->reply_queue, TYPE_HASH("msg_link"));
6512
6513 8958 again:
6514
1/2
✓ Branch 0 taken 898454 times.
✗ Branch 1 not taken.
 898454 while (!xcom_shutdown) {
6515 898454 ep->site = nullptr;
6516
2/4
✓ Branch 0 taken 898454 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 898454 times.
✗ Branch 3 not taken.
 898454 unchecked_replace_pax_msg(&ep->p, pax_msg_new_0(null_synode));
6517
6518
1/2
✓ Branch 0 taken 898454 times.
✗ Branch 1 not taken.
 898454 if (use_buffered_read) {
6519
12/18
✓ Branch 0 taken 898454 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1222410 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 898443 times.
✓ Branch 5 taken 323967 times.
✓ Branch 6 taken 1721 times.
✓ Branch 7 taken 896722 times.
✓ Branch 8 taken 323967 times.
✓ Branch 9 taken 896722 times.
✗ Branch 10 not taken.
✓ Branch 11 taken 323956 times.
✗ Branch 12 not taken.
✓ Branch 13 taken 323956 times.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✓ Branch 16 taken 323956 times.
✓ Branch 17 taken 896722 times.
 1546366 TASK_CALL(buffered_read_msg(ep->rfd, ep->in_buf, ep->p, ep->srv, &n));
6520 } else {
6521 /* purecov: begin deadcode */
6522 ✗ TASK_CALL(read_msg(ep->rfd, ep->p, ep->srv, &n));
6523 /* purecov: end */
6524 }
6525 ADD_DBG(D_NONE, add_synode_event(ep->p->synode);
6526 add_event(EVENT_DUMP_PAD, string_arg("ep->p->from"));
6527 add_event(EVENT_DUMP_PAD, uint_arg(ep->p->from));
6528 add_event(EVENT_DUMP_PAD, string_arg(pax_op_to_str(ep->p->op))););
6529
6530
8/10
✓ Branch 0 taken 624405 times.
✓ Branch 1 taken 272317 times.
✓ Branch 2 taken 624405 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 624405 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 570912 times.
✓ Branch 7 taken 53493 times.
✓ Branch 8 taken 570912 times.
✓ Branch 9 taken 325810 times.
 1521127 if (ep->srv && !ep->srv->invalid && ((int)ep->p->op != (int)client_msg) &&
6531 624405 is_connected(ep->srv->con))
6532
1/2
✓ Branch 0 taken 570912 times.
✗ Branch 1 not taken.
 570912 server_detected(ep->srv);
6533
6534
2/4
✓ Branch 0 taken 896722 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 896722 times.
 896722 if (((int)ep->p->op < (int)client_msg || ep->p->op > LAST_OP)) {
6535 /* invalid operation, ignore message */
6536 ✗ delete_pax_msg(ep->p);
6537 ✗ ep->p = nullptr;
6538 ✗ TASK_YIELD;
6539 ✗ continue;
6540 }
6541
2/2
✓ Branch 0 taken 6135 times.
✓ Branch 1 taken 890587 times.
 896722 if (n <= 0) {
6542 6135 break;
6543 }
6544
2/2
✓ Branch 0 taken 888729 times.
✓ Branch 1 taken 1858 times.
 890587 if (ep->p->op != client_msg) { // Clients have no site
6545
1/2
✓ Branch 0 taken 888729 times.
✗ Branch 1 not taken.
 888729 ep->site = find_site_def(ep->p->synode);
6546 }
6547
6548 /* Handle this connection on a local_server task instead of this
6549 acceptor_learner_task task. */
6550 /* purecov: begin deadcode */
6551
3/4
✓ Branch 0 taken 1858 times.
✓ Branch 1 taken 888729 times.
✓ Branch 2 taken 1858 times.
✗ Branch 3 not taken.
 890587 if (ep->p->op == client_msg && ep->p->a &&
6552
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1858 times.
 1858 ep->p->a->body.c_t == convert_into_local_server_type) {
6553 ✗ if (local_server_is_setup()) {
6554 /* Launch local_server task to handle this connection. */
6555 {
6556 ✗ connection_descriptor *con = (connection_descriptor *)xcom_malloc(
6557 sizeof(connection_descriptor));
6558 ✗ *con = *ep->rfd;
6559 ✗ task_new(local_server, void_arg(con), "local_server",
6560 XCOM_THREAD_DEBUG);
6561 }
6562 }
6563 /* Reply to client:
6564 - OK if local_server task is setup, or
6565 - FAIL otherwise. */
6566 {
6567 ✗ CREATE_REPLY(ep->p);
6568 ✗ reply->op = xcom_client_reply;
6569 ✗ reply->cli_err = local_server_is_setup() ? REQUEST_OK : REQUEST_FAIL;
6570 ✗ SERIALIZE_REPLY(reply);
6571 ✗ replace_pax_msg(&reply, nullptr);
6572 }
6573 ✗ WRITE_REPLY;
6574 ✗ delete_pax_msg(ep->p);
6575 ✗ ep->p = nullptr;
6576 ✗ if (local_server_is_setup()) {
6577 /* Relinquish ownership of the connection. It is now onwed by the
6578 launched local_server task. */
6579 ✗ reset_connection(ep->rfd);
6580 }
6581 /* Terminate this task. */
6582 ✗ TERMINATE;
6583 }
6584 /* purecov: end */
6585
6586 /*
6587 Getting a pointer to the server needs to be done after we have
6588 received a message, since without having received a message, we
6589 cannot know who it is from. We could peek at the message and de‐
6590 serialize the message number and from field, but since the server
6591 does not change, it should be sufficient to cache the server in
6592 the acceptor_learner task. A cleaner solution would have been to
6593 move the timestamps out of the server object, and have a map in‐
6594 dexed by IP/port or UUID to track the timestamps, since this is
6595 common to both the sender_task, reply_handler_task, and the ac‐
6596 ceptor_learner_task.
6597 */
6598
1/2
✓ Branch 0 taken 890587 times.
✗ Branch 1 not taken.
 890587 update_srv(&ep->srv, get_server(ep->site, ep->p->from));
6599 890587 ep->p->refcnt = 1; /* Refcnt from other end is void here */
6600 IFDBG(D_NONE, FN; NDBG(ep->rfd.fd, d); NDBG(task_now(), f);
6601 COPY_AND_FREE_GOUT(dbg_pax_msg(ep->p)););
6602 890587 receive_count[ep->p->op]++;
6603 890587 receive_bytes[ep->p->op] += (uint64_t)n + MSG_HDR_SIZE;
6604 {
6605
3/4
✓ Branch 0 taken 890587 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 827845 times.
✓ Branch 3 taken 62742 times.
 890587 if (get_maxnodes(ep->site) > 0) {
6606 827845 ep->behind = ep->p->synode.msgno < delivered_msg.msgno;
6607 }
6608 ADD_DBG(D_BASE, add_event(EVENT_DUMP_PAD, string_arg("before dispatch "));
6609 add_synode_event(ep->p->synode);
6610 add_event(EVENT_DUMP_PAD, string_arg("ep->p->from"));
6611 add_event(EVENT_DUMP_PAD, uint_arg(ep->p->from));
6612 add_event(EVENT_DUMP_PAD, string_arg(pax_op_to_str(ep->p->op)));
6613 add_event(EVENT_DUMP_PAD,
6614 string_arg(pax_msg_type_to_str(ep->p->msg_type)));
6615 add_event(EVENT_DUMP_PAD, string_arg("is_cached(ep->p->synode)"));
6616 add_event(EVENT_DUMP_PAD, int_arg(is_cached(ep->p->synode)));
6617 add_event(EVENT_DUMP_PAD, string_arg("behind"));
6618 add_event(EVENT_DUMP_PAD, int_arg(ep->behind)););
6619 /* Special treatment to see if synode number is valid. Return no-op if
6620 * not. */
6621
4/4
✓ Branch 0 taken 544197 times.
✓ Branch 1 taken 346390 times.
✓ Branch 2 taken 519961 times.
✓ Branch 3 taken 24236 times.
 890587 if (ep->p->op == read_op || ep->p->op == prepare_op ||
6622
2/2
✓ Branch 0 taken 94403 times.
✓ Branch 1 taken 425558 times.
 519961 ep->p->op == accept_op) {
6623
2/2
✓ Branch 0 taken 430715 times.
✓ Branch 1 taken 34314 times.
 465029 if (ep->site) {
6624 ADD_DBG(
6625 D_BASE, add_event(EVENT_DUMP_PAD, string_arg("ep->p->synode"));
6626 add_synode_event(ep->p->synode);
6627 add_event(EVENT_DUMP_PAD, string_arg("ep->site->start"));
6628 add_synode_event(ep->site->start); add_event(
6629 EVENT_DUMP_PAD, string_arg("ep->site->nodes.node_list_len"));
6630 add_event(EVENT_DUMP_PAD,
6631 uint_arg(ep->site->nodes.node_list_len)););
6632
2/2
✓ Branch 0 taken 1091 times.
✓ Branch 1 taken 429624 times.
 430715 if (ep->p->synode.node >= ep->site->nodes.node_list_len) {
6633 {
6634
2/4
✓ Branch 0 taken 1091 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1091 times.
✗ Branch 3 not taken.
 1091 CREATE_REPLY(ep->p);
6635
1/2
✓ Branch 0 taken 1091 times.
✗ Branch 1 not taken.
 1091 create_noop(reply);
6636 1091 set_learn_type(reply);
6637
3/6
✓ Branch 0 taken 1091 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1091 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1091 times.
✗ Branch 5 not taken.
 1091 SERIALIZE_REPLY(reply);
6638
1/2
✓ Branch 0 taken 1091 times.
✗ Branch 1 not taken.
 1091 delete_pax_msg(reply); /* Deallocate BEFORE potentially blocking
6639 call which will lose value of reply */
6640 }
6641
7/20
✓ Branch 0 taken 1091 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1091 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1091 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 1091 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✓ Branch 9 taken 1091 times.
✗ Branch 10 not taken.
✓ Branch 11 taken 1091 times.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✗ Branch 16 not taken.
✗ Branch 17 not taken.
✗ Branch 18 not taken.
✓ Branch 19 taken 1091 times.
 1091 WRITE_REPLY;
6642 1091 goto again;
6643 }
6644 }
6645 }
6646 /* Reject any message that might compromise the integrity of a consensus
6647 * instance. We do this by not processing any message which may change
6648 * the
6649 * outcome if the consensus instance has been evicted from the cache */
6650 889496 if (harmless(ep->p) || /* Harmless message */
6651
7/8
✓ Branch 0 taken 604703 times.
✓ Branch 1 taken 284793 times.
✓ Branch 2 taken 604703 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 141632 times.
✓ Branch 5 taken 463071 times.
✓ Branch 6 taken 841583 times.
✓ Branch 7 taken 47913 times.
 1031128 is_cached(ep->p->synode) || /* Already in cache */
6652
2/2
✓ Branch 0 taken 93719 times.
✓ Branch 1 taken 47913 times.
 141632 (!ep->behind)) { /* Guard against cache pollution from other nodes
6653 */
6654
6655
2/2
✓ Branch 0 taken 838358 times.
✓ Branch 1 taken 3225 times.
 841583 if (should_poll_cache(ep->p->op)) {
6656
6/18
✓ Branch 0 taken 838358 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 838358 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 838358 times.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✓ Branch 7 taken 838358 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 838358 times.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✗ Branch 16 not taken.
✓ Branch 17 taken 838358 times.
 838358 TASK_CALL(wait_for_cache(&pm, ep->p->synode, 10));
6657
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 838358 times.
 838358 if (!pm) continue; /* Could not get a machine, discarding message. */
6658 }
6659
6660
1/2
✓ Branch 0 taken 841583 times.
✗ Branch 1 not taken.
 841583 dispatch_op(ep->site, ep->p, &ep->reply_queue);
6661
6662 /* Send replies on same fd */
6663
2/2
✓ Branch 0 taken 220894 times.
✓ Branch 1 taken 841583 times.
 1062477 while (!link_empty(&ep->reply_queue)) {
6664 {
6665 msg_link *reply =
6666
1/2
✓ Branch 0 taken 220894 times.
✗ Branch 1 not taken.
 220894 (msg_link *)(link_extract_first(&ep->reply_queue));
6667 IFDBG(D_DISPATCH, FN; PTREXP(reply);
6668 COPY_AND_FREE_GOUT(dbg_linkage(&ep->reply_queue));
6669 COPY_AND_FREE_GOUT(dbg_msg_link(reply));
6670 COPY_AND_FREE_GOUT(dbg_pax_msg(reply->p)););
6671
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 220894 times.
 220894 assert(reply->p);
6672
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 220894 times.
 220894 assert(reply->p->refcnt > 0);
6673 IFDBG(D_DISPATCH, FN; STRLIT("serialize "); PTREXP(reply));
6674
3/6
✓ Branch 0 taken 220894 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 220894 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 220894 times.
✗ Branch 5 not taken.
 220894 SERIALIZE_REPLY(reply->p);
6675
1/2
✓ Branch 0 taken 220894 times.
✗ Branch 1 not taken.
 220894 msg_link_delete(&reply); /* Deallocate BEFORE potentially blocking
6676 call which will lose value of reply */
6677 }
6678
7/20
✓ Branch 0 taken 220894 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 220894 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 220894 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 220894 times.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✓ Branch 9 taken 220894 times.
✗ Branch 10 not taken.
✓ Branch 11 taken 220894 times.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✗ Branch 16 not taken.
✗ Branch 17 not taken.
✗ Branch 18 not taken.
✓ Branch 19 taken 220894 times.
 220894 WRITE_REPLY;
6679 }
6680 } else {
6681 IFDBG(D_EXEC, FN; STRLIT("rejecting ");
6682 STRLIT(pax_op_to_str(ep->p->op)); NDBG(ep->p->from, d);
6683 NDBG(ep->p->to, d); SYCEXP(ep->p->synode);
6684 BALCEXP(ep->p->proposal));
6685
1/2
✓ Branch 0 taken 47913 times.
✗ Branch 1 not taken.
 47913 if (/* xcom_booted() && */ ep->behind) {
6686
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 47913 times.
 47913 if (/*ep->p->op == prepare_op && */ was_removed_from_cache(
6687
1/2
✓ Branch 0 taken 47913 times.
✗ Branch 1 not taken.
 47913 ep->p->synode)) {
6688 IFDBG(D_NONE, FN; STRLIT("send_die ");
6689 STRLIT(pax_op_to_str(ep->p->op)); NDBG(ep->p->from, d);
6690 NDBG(ep->p->to, d); SYCEXP(ep->p->synode);
6691 BALCEXP(ep->p->proposal));
6692 ✗ if (get_maxnodes(ep->site) > 0) {
6693 {
6694 ✗ pax_msg *np = nullptr;
6695 ✗ np = pax_msg_new(ep->p->synode, ep->site);
6696 ✗ np->op = die_op;
6697 ✗ SERIALIZE_REPLY(np);
6698 IFDBG(D_NONE, FN; STRLIT("sending die_op to node ");
6699 NDBG(np->to, d); SYCEXP(executed_msg); SYCEXP(max_synode);
6700 SYCEXP(np->synode));
6701 ✗ delete_pax_msg(np); /* Deallocate BEFORE potentially blocking
6702 call which will lose value of np */
6703 }
6704 ✗ WRITE_REPLY;
6705 }
6706 }
6707 }
6708 }
6709 }
6710 /* TASK_YIELD; */
6711 }
6712
6713 ✗ FINALLY
6714 IFDBG(D_BUG, FN; STRLIT(" shutdown "); NDBG(ep->rfd.fd, d);
6715 NDBG(task_now(), f));
6716
3/6
✓ Branch 0 taken 7856 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 7856 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 7856 times.
 7856 if (ep->reply_queue.suc && !link_empty(&ep->reply_queue))
6717 ✗ empty_msg_list(&ep->reply_queue);
6718
1/2
✓ Branch 0 taken 7856 times.
✗ Branch 1 not taken.
 7856 unchecked_replace_pax_msg(&ep->p, nullptr);
6719
1/2
✓ Branch 0 taken 7856 times.
✗ Branch 1 not taken.
 7856 shutdown_connection(ep->rfd);
6720 7856 free(ep->rfd);
6721 IFDBG(D_NONE, FN; NDBG(xcom_shutdown, d));
6722
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7856 times.
 7856 if (ep->buf) X_FREE(ep->buf);
6723 7856 free(ep->in_buf);
6724
6725 /* Unref srv to avoid leak */
6726
1/2
✓ Branch 0 taken 7856 times.
✗ Branch 1 not taken.
 7856 update_srv(&ep->srv, nullptr);
6727
6728 IFDBG(D_BUG, FN; STRLIT(" shutdown completed"); NDBG(ep->rfd.fd, d);
6729 NDBG(task_now(), f));
6730
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 7856 times.
✓ Branch 2 taken 7856 times.
✗ Branch 3 not taken.
 7856 TASK_END;
6731 }
6732
6733 /* Reply handler task */
6734
6735 static void server_handle_need_snapshot(server *srv, site_def const *s,
6736 node_no node);
6737
6738 338069 int reply_handler_task(task_arg arg) {
6739 DECL_ENV
6740 server *s;
6741 pax_msg *reply;
6742 double dtime;
6743 3014 ENV_INIT
6744 3014 END_ENV_INIT
6745 END_ENV;
6746
6747 338069 int64_t n{0};
6748
9/15
✓ Branch 0 taken 3014 times.
✓ Branch 1 taken 9480 times.
✓ Branch 2 taken 105732 times.
✓ Branch 3 taken 219843 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 3014 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 3014 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✓ Branch 10 taken 3014 times.
✓ Branch 11 taken 3014 times.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
✓ Branch 14 taken 3014 times.
 338069 TASK_BEGIN
6749
6750 3014 ep->dtime = INITIAL_CONNECT_WAIT; /* Initial wait is short, to avoid
6751 unnecessary waiting */
6752 3014 ep->s = (server *)get_void_arg(arg);
6753
1/2
✓ Branch 0 taken 3014 times.
✗ Branch 1 not taken.
 3014 srv_ref(ep->s);
6754 3014 ep->reply = nullptr;
6755
6756
1/2
✓ Branch 0 taken 224867 times.
✗ Branch 1 not taken.
 224867 while (!xcom_shutdown) {
6757
2/2
✓ Branch 0 taken 9480 times.
✓ Branch 1 taken 223564 times.
 233044 while (!is_connected(ep->s->con)) {
6758 IFDBG(D_NONE, FN; STRLIT("waiting for connection"));
6759
6/10
✓ Branch 0 taken 9480 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 9480 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 9480 times.
✓ Branch 6 taken 9480 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 1303 times.
✓ Branch 9 taken 8177 times.
 18960 TASK_DELAY(ep->dtime);
6760
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 8177 times.
 8177 if (xcom_shutdown) {
6761 ✗ TERMINATE;
6762 }
6763 8177 ep->dtime += CONNECT_WAIT_INCREASE; /* Increase wait time for next try */
6764
2/2
✓ Branch 0 taken 386 times.
✓ Branch 1 taken 7791 times.
 8177 if (ep->dtime > MAX_CONNECT_WAIT) {
6765 386 ep->dtime = MAX_CONNECT_WAIT;
6766 }
6767 }
6768 223564 ep->dtime = INITIAL_CONNECT_WAIT;
6769 {
6770
2/4
✓ Branch 0 taken 223564 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 223564 times.
✗ Branch 3 not taken.
 223564 unchecked_replace_pax_msg(&ep->reply, pax_msg_new_0(null_synode));
6771
6772 ADD_DBG(D_NONE, add_event(EVENT_DUMP_PAD, string_arg("ep->s->con.fd"));
6773 add_event(EVENT_DUMP_PAD, int_arg(ep->s->con.fd)););
6774
12/18
✓ Branch 0 taken 223564 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 329296 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 223553 times.
✓ Branch 5 taken 105743 times.
✓ Branch 6 taken 1700 times.
✓ Branch 7 taken 221853 times.
✓ Branch 8 taken 105743 times.
✓ Branch 9 taken 221853 times.
✗ Branch 10 not taken.
✓ Branch 11 taken 105732 times.
✗ Branch 12 not taken.
✓ Branch 13 taken 105732 times.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✓ Branch 16 taken 105732 times.
✓ Branch 17 taken 221853 times.
 435028 TASK_CALL(read_msg(ep->s->con, ep->reply, ep->s, &n));
6775 ADD_DBG(D_NONE, add_event(EVENT_DUMP_PAD, string_arg("ep->s->con.fd"));
6776 add_event(EVENT_DUMP_PAD, int_arg(ep->s->con.fd)););
6777 221853 ep->reply->refcnt = 1; /* Refcnt from other end is void here */
6778
2/2
✓ Branch 0 taken 2010 times.
✓ Branch 1 taken 219843 times.
 221853 if (n <= 0) {
6779
1/2
✓ Branch 0 taken 2010 times.
✗ Branch 1 not taken.
 2010 shutdown_connection(ep->s->con);
6780 2010 continue;
6781 }
6782 219843 receive_bytes[ep->reply->op] += (uint64_t)n + MSG_HDR_SIZE;
6783 }
6784 IFDBG(D_NONE, FN; NDBG(ep->s->con.fd, d); NDBG(task_now(), f);
6785 COPY_AND_FREE_GOUT(dbg_pax_msg(ep->reply)););
6786 219843 receive_count[ep->reply->op]++;
6787
6788 ADD_DBG(D_NONE, add_synode_event(ep->reply->synode);
6789 add_event(EVENT_DUMP_PAD, string_arg("ep->reply->from"));
6790 add_event(EVENT_DUMP_PAD, uint_arg(ep->reply->from));
6791 add_event(EVENT_DUMP_PAD, string_arg(pax_op_to_str(ep->reply->op)));
6792 add_event(EVENT_DUMP_PAD, string_arg("get_site_def()->boot_key"));
6793 add_synode_event(get_site_def()->boot_key););
6794 /* Special test for need_snapshot, since node and site may not be
6795 * consistent
6796 */
6797
4/4
✓ Branch 0 taken 1318 times.
✓ Branch 1 taken 218525 times.
✓ Branch 2 taken 1318 times.
✓ Branch 3 taken 218525 times.
 221161 if (ep->reply->op == need_boot_op &&
6798
3/6
✓ Branch 0 taken 1318 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1318 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1318 times.
✗ Branch 5 not taken.
 1318 !synode_eq(get_site_def()->boot_key, null_synode)) {
6799 1318 pax_msg *p = ep->reply;
6800
6801 ADD_DBG(D_BASE,
6802 add_event(EVENT_DUMP_PAD,
6803 string_arg("calling server_handle_need_snapshot")););
6804
3/6
✓ Branch 0 taken 1318 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1318 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1318 times.
✗ Branch 5 not taken.
 1318 if (should_handle_need_boot(find_site_def(p->synode), p)) {
6805
2/4
✓ Branch 0 taken 1318 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1318 times.
✗ Branch 3 not taken.
 1318 server_handle_need_snapshot(ep->s, find_site_def(p->synode), p->from);
6806 /* Wake senders waiting to connect, since new node has appeared */
6807
1/2
✓ Branch 0 taken 1318 times.
✗ Branch 1 not taken.
 1318 wakeup_sender();
6808 } else {
6809 ✗ ep->s->invalid = 1;
6810 }
6811 } else {
6812 /* We only handle messages from this connection if the server is valid.
6813 */
6814
1/2
✓ Branch 0 taken 218525 times.
✗ Branch 1 not taken.
 218525 if (ep->s->invalid == 0)
6815
2/4
✓ Branch 0 taken 218525 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 218525 times.
✗ Branch 3 not taken.
 218525 dispatch_op(find_site_def(ep->reply->synode), ep->reply, nullptr);
6816 }
6817
3/6
✗ Branch 0 not taken.
✓ Branch 1 taken 219843 times.
✓ Branch 2 taken 219843 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 219843 times.
 439686 TASK_YIELD;
6818 }
6819
6820 ✗ FINALLY
6821
1/2
✓ Branch 0 taken 3003 times.
✗ Branch 1 not taken.
 3003 replace_pax_msg(&ep->reply, nullptr);
6822
6823
1/2
✓ Branch 0 taken 3003 times.
✗ Branch 1 not taken.
 3003 shutdown_connection(ep->s->con);
6824 3003 ep->s->reply_handler = nullptr;
6825 IFDBG(D_BUG, FN; STRLIT(" shutdown "); NDBG(ep->s->con.fd, d);
6826 NDBG(task_now(), f));
6827
1/2
✓ Branch 0 taken 3003 times.
✗ Branch 1 not taken.
 3003 srv_unref(ep->s);
6828
6829
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 3003 times.
✓ Branch 2 taken 3003 times.
✗ Branch 3 not taken.
 3003 TASK_END;
6830 }
6831
6832 /* purecov: begin deadcode */
6833 ✗ void xcom_sleep(unsigned int seconds) {
6834 #if defined(_WIN32)
6835 Sleep((DWORD)seconds * 1000); /* windows sleep takes milliseconds */
6836 #else
6837 ✗ sleep(seconds);
6838 #endif
6839 }
6840 /* purecov: end */
6841
6842 /*
6843 * Get a unique long as the basis for XCom group id creation.
6844 *
6845 * NOTE:
6846 * As there is no gethostid() on win, we use seconds since epoch instead,
6847 * so it might fail if you try simultaneous create sites at the same second.
6848 */
6849
6850 #ifndef _WIN32
6851 #include <sys/utsname.h>
6852 #endif
6853
6854 3423 long xcom_unique_long(void) {
6855 #if defined(_WIN32)
6856 __time64_t ltime;
6857
6858 _time64(&ltime);
6859 return (long)(ltime ^ GetCurrentProcessId());
6860 #else
6861 struct utsname buf;
6862 3423 uname(&buf);
6863 3423 long id = (long)fnv_hash((unsigned char *)&buf, sizeof(buf), 0);
6864 3423 return id ^ getpid();
6865 #endif
6866 }
6867
6868 5282 app_data_ptr init_config_with_group(app_data *a, node_list *nl, cargo_type type,
6869 uint32_t group_id) {
6870 5282 init_app_data(a);
6871 5282 a->app_key.group_id = a->group_id = group_id;
6872 5282 a->body.c_t = type;
6873 5282 init_node_list(nl->node_list_len, nl->node_list_val, &a->body.app_u_u.nodes);
6874 5282 return a;
6875 }
6876
6877 9 app_data_ptr init_set_event_horizon_msg(app_data *a, uint32_t group_id,
6878 xcom_event_horizon event_horizon) {
6879 9 init_app_data(a);
6880 9 a->app_key.group_id = a->group_id = group_id;
6881 9 a->body.c_t = set_event_horizon_type;
6882 9 a->body.app_u_u.event_horizon = event_horizon;
6883 9 return a;
6884 }
6885
6886 185 app_data_ptr init_get_msg(app_data *a, uint32_t group_id, cargo_type const t) {
6887 185 init_app_data(a);
6888 185 a->app_key.group_id = a->group_id = group_id;
6889 185 a->body.c_t = t;
6890 185 return a;
6891 }
6892
6893 85 app_data_ptr init_get_leaders_msg(app_data *a, uint32_t group_id) {
6894 85 return init_get_msg(a, group_id, get_leaders_type);
6895 }
6896
6897 99 app_data_ptr init_get_event_horizon_msg(app_data *a, uint32_t group_id) {
6898 99 return init_get_msg(a, group_id, get_event_horizon_type);
6899 }
6900
6901 76202 app_data_ptr init_app_msg(app_data *a, char *payload, u_int payload_size) {
6902 76202 init_app_data(a);
6903 76202 a->body.c_t = app_type;
6904 76202 a->body.app_u_u.data.data_val = payload; /* Takes ownership of payload. */
6905 76202 a->body.app_u_u.data.data_len = payload_size;
6906 76202 return a;
6907 }
6908
6909 1 static app_data_ptr init_get_synode_app_data_msg(
6910 app_data *a, uint32_t group_id, synode_no_array *const synodes) {
6911 1 init_get_msg(a, group_id, get_synode_app_data_type);
6912 /* Move synodes (as in C++ move semantics) into a->body.app_u_u.synodes. */
6913 1 synode_array_move(&a->body.app_u_u.synodes, synodes);
6914 1 return a;
6915 }
6916
6917 3 app_data_ptr init_set_cache_size_msg(app_data *a, uint64_t cache_limit) {
6918 3 init_app_data(a);
6919 3 a->body.c_t = set_cache_limit;
6920 3 a->body.app_u_u.cache_limit = cache_limit;
6921 3 return a;
6922 }
6923
6924 ✗ app_data_ptr init_convert_into_local_server_msg(app_data *a) {
6925 ✗ init_app_data(a);
6926 ✗ a->body.c_t = convert_into_local_server_type;
6927 ✗ return a;
6928 }
6929
6930 1318 static void server_send_snapshot(server *srv, site_def const *s,
6931 gcs_snapshot *gcs_snap, node_no node) {
6932
2/4
✓ Branch 0 taken 1318 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1318 times.
✗ Branch 3 not taken.
 1318 pax_msg *p = pax_msg_new(gcs_snap->log_start, get_site_def());
6933
1/2
✓ Branch 0 taken 1318 times.
✗ Branch 1 not taken.
 1318 ref_msg(p);
6934 1318 p->op = gcs_snapshot_op;
6935 1318 p->gcs_snap = gcs_snap;
6936
2/4
✓ Branch 0 taken 1318 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1318 times.
✗ Branch 3 not taken.
 1318 send_msg(srv, s->nodeno, node, get_group_id(s), p);
6937
1/2
✓ Branch 0 taken 1318 times.
✗ Branch 1 not taken.
 1318 unref_msg(&p);
6938 1318 }
6939
6940 1318 static void server_push_log(server *srv, synode_no push, node_no node) {
6941 1318 site_def const *s = get_site_def();
6942
2/4
✓ Branch 0 taken 1318 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1318 times.
✗ Branch 3 not taken.
 1318 if (srv && s) {
6943
2/2
✓ Branch 0 taken 57319 times.
✓ Branch 1 taken 1318 times.
 58637 while (!synode_gt(push, get_max_synode())) {
6944
2/2
✓ Branch 0 taken 54155 times.
✓ Branch 1 taken 3164 times.
 57319 if (is_cached(push)) {
6945 /* Need to clone message here since pax_machine may be re-used while
6946 * message is sent */
6947 54155 pax_machine *p = get_cache_no_touch(push, FALSE);
6948
2/2
✓ Branch 0 taken 43805 times.
✓ Branch 1 taken 10350 times.
 54155 if (pm_finished(p)) {
6949
1/2
✓ Branch 0 taken 43805 times.
✗ Branch 1 not taken.
 43805 pax_msg *pm = clone_pax_msg(p->learner.msg);
6950
1/2
✓ Branch 0 taken 43805 times.
✗ Branch 1 not taken.
 43805 if (pm != nullptr) {
6951
1/2
✓ Branch 0 taken 43805 times.
✗ Branch 1 not taken.
 43805 ref_msg(pm);
6952 43805 pm->op = recover_learn_op;
6953 IFDBG(D_NONE, FN; PTREXP(srv); PTREXP(s););
6954
2/4
✓ Branch 0 taken 43805 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 43805 times.
✗ Branch 3 not taken.
 43805 send_msg(srv, s->nodeno, node, get_group_id(s), pm);
6955
1/2
✓ Branch 0 taken 43805 times.
✗ Branch 1 not taken.
 43805 unref_msg(&pm);
6956 }
6957 }
6958 }
6959 57319 push = incr_synode(push);
6960 }
6961 }
6962 1318 }
6963
6964 /* purecov: begin deadcode */
6965 ✗ static void reply_push_log(synode_no push, linkage *reply_queue) {
6966 ✗ while (!synode_gt(push, get_max_synode())) {
6967 ✗ if (is_cached(push)) {
6968 /* Need to clone message here since pax_machine may be re-used while
6969 * message is sent */
6970 ✗ pax_machine *p = get_cache_no_touch(push, FALSE);
6971 ✗ if (pm_finished(p)) {
6972 ✗ pax_msg *reply = clone_pax_msg(p->learner.msg);
6973 ✗ ref_msg(reply);
6974 ✗ reply->op = recover_learn_op;
6975 {
6976 ✗ msg_link *msg_x = msg_link_new(reply, reply->from);
6977 IFDBG(D_NONE, FN; PTREXP(msg_x));
6978 ✗ link_into(&(msg_x->l), reply_queue);
6979 }
6980 ✗ replace_pax_msg(&reply, nullptr);
6981 ✗ unref_msg(&reply);
6982 }
6983 }
6984 ✗ push = incr_synode(push);
6985 }
6986 }
6987 /* purecov: end */
6988
6989 static app_snap_getter get_app_snap_cb;
6990 static app_snap_handler handle_app_snap_cb;
6991
6992 1318 static gcs_snapshot *create_snapshot() {
6993 1318 gcs_snapshot *gs = nullptr;
6994
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1318 times.
 1318 if (get_app_snap_cb) {
6995 /* purecov: begin deadcode */
6996 ✗ blob app_snap = {{0, nullptr}}; /* Initialize in case get_app_snap_cb does
6997 not assign a value */
6998 ✗ synode_no app_lsn = get_app_snap_cb(&app_snap);
6999
7000 /* We have a valid callback, abort if it did not return anything */
7001 ✗ if (app_snap.data.data_len == 0) {
7002 ADD_DBG(D_BASE,
7003 add_event(EVENT_DUMP_PAD, string_arg("no data, return")););
7004 ✗ return nullptr;
7005 }
7006 ✗ gs = export_config();
7007 ✗ if (!gs) return nullptr;
7008 ADD_DBG(D_BASE, add_event(EVENT_DUMP_PAD, string_arg("export config ok")););
7009 ✗ gs->app_snap = app_snap;
7010 IFDBG(D_BUG, FN; SYCEXP(app_lsn); SYCEXP(gs->log_start);
7011 SYCEXP(gs->log_end));
7012
7013 /* Set starting point of log to match the snapshot */
7014 /* If we have a valid synode from application snapshot, see if it should
7015 * be used */
7016 ✗ if (!synode_eq(null_synode, app_lsn)) {
7017 /* If log_start is null_synode, always use valid synode from application
7018 * snapshot */
7019 ✗ if (synode_eq(null_synode, gs->log_start) ||
7020 ✗ !synode_gt(app_lsn, gs->log_start)) {
7021 ✗ gs->log_start = app_lsn;
7022 IFDBG(D_BUG, FN; STRLIT("using "); SYCEXP(app_lsn));
7023 }
7024 }
7025 ADD_DBG(D_BASE, add_event(EVENT_DUMP_PAD, string_arg("gs->log_start"));
7026 add_synode_event(gs->log_start);
7027 add_event(EVENT_DUMP_PAD, string_arg("gs->log_end"));
7028 add_synode_event(gs->log_end););
7029 /* purecov: end */
7030 } else {
7031 1318 gs = export_config();
7032
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1318 times.
 1318 if (!gs) return nullptr;
7033 ADD_DBG(D_BASE, add_event(EVENT_DUMP_PAD, string_arg("export config ok")););
7034
1/2
✓ Branch 0 taken 1318 times.
✗ Branch 1 not taken.
 1318 if (!synode_eq(null_synode, last_config_modification_id)) {
7035 /* No valid valid synode from application snapshot, use
7036 * last_config_modification_id if not null_synode */
7037 1318 gs->log_start = last_config_modification_id;
7038 IFDBG(D_BUG, FN; STRLIT("using "); SYCEXP(last_config_modification_id));
7039 }
7040 IFDBG(D_BUG, FN; SYCEXP(gs->log_start); SYCEXP(gs->log_end));
7041 ADD_DBG(D_BASE, add_event(EVENT_DUMP_PAD, string_arg("gs->log_start"));
7042 add_synode_event(gs->log_start);
7043 add_event(EVENT_DUMP_PAD, string_arg("gs->log_end"));
7044 add_synode_event(gs->log_end););
7045 }
7046 IFDBG(D_BUG, FN; SYCEXP(gs->log_start); SYCEXP(gs->log_end));
7047 1318 return gs;
7048 }
7049
7050 /* purecov: begin deadcode */
7051 ✗ static void handle_need_snapshot(linkage *reply_queue, pax_msg *pm) {
7052 ✗ gcs_snapshot *gs = create_snapshot();
7053 ✗ if (gs) {
7054 ✗ pax_msg *reply = clone_pax_msg(pm);
7055 ✗ ref_msg(reply);
7056 ✗ reply->op = gcs_snapshot_op;
7057 ✗ reply->gcs_snap = gs;
7058 {
7059 ✗ msg_link *msg_x = msg_link_new(reply, reply->from);
7060 IFDBG(D_NONE, FN; PTREXP(msg_x));
7061 ✗ link_into(&(msg_x->l), reply_queue);
7062 }
7063 ✗ unref_msg(&reply);
7064 IFDBG(D_NONE, FN; STRLIT("sent snapshot"););
7065 ✗ reply_push_log(gs->log_start, reply_queue);
7066 ✗ send_global_view();
7067 }
7068 }
7069 /* purecov: end */
7070
7071 static task_env *x_timer = nullptr;
7072
7073 /* Timer for use with the xcom FSM. Will deliver x_fsm_timeout */
7074 ✗ static int xcom_timer(task_arg arg) {
7075 DECL_ENV
7076 double t;
7077 ✗ ENV_INIT
7078 ✗ END_ENV_INIT
7079 END_ENV;
7080
7081 ✗ TASK_BEGIN
7082
7083 ✗ ep->t = get_double_arg(arg);
7084 ✗ TASK_DELAY(ep->t);
7085 ✗ XCOM_FSM(x_fsm_timeout, double_arg(ep->t));
7086 ✗ FINALLY
7087 ✗ if (stack == x_timer) set_task(&x_timer, nullptr);
7088 IFDBG(D_CONS, FN; STRLIT(" timeout "));
7089 ✗ TASK_END;
7090 }
7091
7092 /* Stop the xcom FSM timer */
7093 2215 static void stop_x_timer() {
7094
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2215 times.
 2215 if (x_timer) {
7095 ✗ task_terminate(x_timer);
7096 ✗ set_task(&x_timer, nullptr);
7097 }
7098 2215 }
7099
7100 /* Start the xcom FSM timer */
7101 ✗ static void start_x_timer(double t) {
7102 ✗ stop_x_timer();
7103 ✗ set_task(&x_timer, task_new(xcom_timer, double_arg(t), "xcom_timer",
7104 XCOM_THREAD_DEBUG));
7105 }
7106
7107 /* Deliver x_fsm_complete to xcom FSM */
7108 /* purecov: begin deadcode */
7109 ✗ static int x_fsm_completion_task(task_arg arg) {
7110 DECL_ENV
7111 int dummy;
7112 ✗ ENV_INIT
7113 ✗ END_ENV_INIT
7114 END_ENV;
7115
7116 ✗ TASK_BEGIN
7117
7118 (void)
7119 arg;
7120 ✗ XCOM_FSM(x_fsm_complete, null_arg);
7121 ✗ FINALLY
7122 IFDBG(D_FSM, FN; STRLIT(" delivered "));
7123 ✗ TASK_END;
7124 }
7125 /* purecov: end */
7126
7127 /* Send x_fsm_complete to xcom FSM in the context of the xcom thread. The
7128 * calling thread and the xcom thread must be in a rendezvous. Using a task to
7129 * deliver a message is an abstraction inversion, but it's the simplest
7130 * solution until we get a proper queue-based communication system going. */
7131 /* purecov: begin deadcode */
7132 ✗ void send_x_fsm_complete() {
7133 ✗ task_new(x_fsm_completion_task, null_arg, "x_fsm_completion_task",
7134 XCOM_THREAD_DEBUG);
7135 }
7136 /* purecov: end */
7137
7138 1318 static void server_handle_need_snapshot(server *srv, site_def const *s,
7139 node_no node) {
7140
2/4
✓ Branch 0 taken 1318 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1318 times.
✗ Branch 3 not taken.
 1318 G_INFO("Received an XCom snapshot request from %s:%d", srv->srv, srv->port);
7141 1318 gcs_snapshot *gs = create_snapshot();
7142
7143
1/2
✓ Branch 0 taken 1318 times.
✗ Branch 1 not taken.
 1318 if (gs) {
7144 1318 server_send_snapshot(srv, s, gs, node);
7145 IFDBG(D_NONE, FN; STRLIT("sent snapshot"););
7146
2/4
✓ Branch 0 taken 1318 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1318 times.
✗ Branch 3 not taken.
 1318 G_INFO("XCom snapshot sent to %s:%d", srv->srv, srv->port);
7147 1318 server_push_log(srv, gs->log_start, node);
7148 1318 send_global_view();
7149 }
7150 1318 }
7151
7152 #define X(b) #b
7153 const char *xcom_actions_name[] = {x_actions};
7154 #undef X
7155
7156 static int snapshots[NSERVERS];
7157
7158 /* Note that we have received snapshot from node */
7159 1367 static void note_snapshot(node_no node) {
7160
1/2
✓ Branch 0 taken 1367 times.
✗ Branch 1 not taken.
 1367 if (node != VOID_NODE_NO) {
7161 1367 snapshots[node] = 1;
7162 }
7163 1367 }
7164
7165 /* Reset set of received snapshots */
7166 3423 static void reset_snapshot_mask() {
7167 int i;
7168
2/2
✓ Branch 0 taken 342300 times.
✓ Branch 1 taken 3423 times.
 345723 for (i = 0; i < NSERVERS; i++) {
7169 342300 snapshots[i] = 0;
7170 }
7171 3423 }
7172
7173 /* See if we have got a snapshot from every node */
7174 ✗ static int got_all_snapshots() {
7175 node_no i;
7176 ✗ node_no max = get_maxnodes(get_site_def());
7177 ✗ if (0 == max) {
7178 ✗ return 0;
7179 }
7180 ✗ for (i = 0; i < max; i++) {
7181 ✗ if (!snapshots[i]) {
7182 ✗ return 0;
7183 }
7184 }
7185 ✗ return 1;
7186 }
7187
7188 static synode_no log_start_max; /* Initialized by xcom_fsm */
7189 static synode_no log_end_max; /* Initialized by xcom_fsm */
7190
7191 /* See if this snapshot is better than what we already have */
7192 /* purecov: begin deadcode */
7193 ✗ static int better_snapshot(gcs_snapshot *gcs) {
7194 ✗ synode_no boot_key = config_max_boot_key(gcs);
7195 ✗ return synode_gt(boot_key, get_site_def()->boot_key) ||
7196 ✗ (synode_eq(boot_key, get_site_def()->boot_key) &&
7197 ✗ (synode_gt(gcs->log_start, log_start_max) ||
7198 ✗ (synode_eq(gcs->log_start, log_start_max) &&
7199 ✗ synode_gt(gcs->log_end, log_end_max))));
7200 }
7201 /* purecov: end */
7202
7203 /* Install snapshot */
7204 1367 static void handle_x_snapshot(gcs_snapshot *gcs) {
7205
2/4
✓ Branch 0 taken 1367 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1367 times.
✗ Branch 3 not taken.
 1367 G_INFO(
7206 "Installing requested snapshot. Importing all incoming configurations.");
7207 1367 import_config(gcs);
7208
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1367 times.
 1367 if (get_nodeno(get_site_def()) == VOID_NODE_NO) {
7209 IFDBG(D_BASE, FN; STRLIT("Not member of site, not executing log"));
7210 ✗ gcs->log_end =
7211 gcs->log_start; /* Avoid executing log if not member of site */
7212 }
7213
1/2
✓ Branch 0 taken 1367 times.
✗ Branch 1 not taken.
 1367 if (handle_app_snap_cb)
7214 1367 handle_app_snap_cb(&gcs->app_snap, gcs->log_start, gcs->log_end);
7215 1367 set_max_synode(gcs->log_end);
7216 1367 set_executed_msg(incr_synode(gcs->log_start));
7217 1367 log_start_max = gcs->log_start;
7218 1367 log_end_max = gcs->log_end;
7219
7220 1367 set_last_received_config(get_highest_boot_key(gcs));
7221
7222
4/8
✓ Branch 0 taken 1367 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1367 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1367 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 1367 times.
✗ Branch 7 not taken.
 1367 G_INFO("Finished snapshot installation. My node number is %d",
7223 get_nodeno(get_site_def()));
7224
7225 IFDBG(D_BUG, FN; SYCEXP(gcs->log_start); SYCEXP(gcs->log_end);
7226 SYCEXP(last_config_modification_id); SYCEXP(executed_msg););
7227 1367 }
7228
7229 /* Note that we have received snapshot, and install if better than old */
7230 /* purecov: begin deadcode */
7231 ✗ static void update_best_snapshot(gcs_snapshot *gcs) {
7232 ✗ if (get_site_def() == nullptr || better_snapshot(gcs)) {
7233 ✗ handle_x_snapshot(gcs);
7234 }
7235 }
7236 /* purecov: end */
7237
7238 /* Send need_boot_op to all nodes in current config */
7239 /* purecov: begin deadcode */
7240 ✗ static void send_need_boot() {
7241 ✗ pax_msg *p = pax_msg_new_0(null_synode);
7242 ✗ ref_msg(p);
7243 ✗ p->synode = get_site_def()->start;
7244 ✗ p->op = need_boot_op;
7245 ✗ send_to_all_except_self(get_site_def(), p, "need_boot_op");
7246 ✗ unref_msg(&p);
7247 }
7248 /* purecov: end */
7249
7250 /* Set log_end of snapshot based on log_end in snapshot and max synode */
7251 2685 void set_log_end(gcs_snapshot *gcs) {
7252
2/2
✓ Branch 0 taken 1318 times.
✓ Branch 1 taken 1367 times.
 2685 if (synode_gt(get_max_synode(), gcs->log_end)) {
7253 1318 gcs->log_end = get_max_synode();
7254 }
7255 2685 }
7256
7257 struct xcom_fsm_state;
7258 typedef struct xcom_fsm_state xcom_fsm_state;
7259
7260 /* Function pointer corresponding to a state. Return 1 if execution should
7261 * continue, 0 otherwise */
7262 typedef int (*xcom_fsm_fp)(xcom_actions action, task_arg fsmargs,
7263 xcom_fsm_state *ctxt);
7264
7265 /* Function pointer and name */
7266 struct xcom_fsm_state {
7267 xcom_fsm_fp state_fp;
7268 char const *state_name;
7269 };
7270
7271 #define X_FSM_STATE(s) \
7272 { s, #s }
7273 #define SET_X_FSM_STATE(s) \
7274 do { \
7275 ctxt->state_fp = s; \
7276 ctxt->state_name = #s; \
7277 } while (0)
7278
7279 /* The state functions/thunks */
7280 static int xcom_fsm_init(xcom_actions action, task_arg fsmargs,
7281 xcom_fsm_state *ctxt);
7282 static int xcom_fsm_start_enter(xcom_actions action, task_arg fsmargs,
7283 xcom_fsm_state *ctxt);
7284 static int xcom_fsm_start(xcom_actions action, task_arg fsmargs,
7285 xcom_fsm_state *ctxt);
7286 static int xcom_fsm_snapshot_wait_enter(xcom_actions action, task_arg fsmargs,
7287 xcom_fsm_state *ctxt);
7288 static int xcom_fsm_snapshot_wait(xcom_actions action, task_arg fsmargs,
7289 xcom_fsm_state *ctxt);
7290 static int xcom_fsm_recover_wait_enter(xcom_actions action, task_arg fsmargs,
7291 xcom_fsm_state *ctxt);
7292 static int xcom_fsm_recover_wait(xcom_actions action, task_arg fsmargs,
7293 xcom_fsm_state *ctxt);
7294 static int xcom_fsm_run_enter(xcom_actions action, task_arg fsmargs,
7295 xcom_fsm_state *ctxt);
7296 static int xcom_fsm_run(xcom_actions action, task_arg fsmargs,
7297 xcom_fsm_state *ctxt);
7298
7299 /* You are in a twisting maze of little functions ... */
7300
7301 /* init state */
7302 1215 static int xcom_fsm_init(xcom_actions action, task_arg fsmargs,
7303 xcom_fsm_state *ctxt) {
7304 (void)action;
7305 (void)fsmargs;
7306 IFDBG(D_NONE, FN;);
7307 /* Initialize basic xcom data */
7308 1215 xcom_thread_init();
7309 1215 SET_X_FSM_STATE(xcom_fsm_start_enter);
7310 1215 return 1;
7311 }
7312
7313 /* start_enter state */
7314 3423 static int xcom_fsm_start_enter(xcom_actions action, task_arg fsmargs,
7315 xcom_fsm_state *ctxt) {
7316 (void)action;
7317 (void)fsmargs;
7318 /* push_dbg(D_DETECT | D_FSM | D_FILEOP | D_CONS | D_BASE | D_TRANSPORT);
7319 */
7320 3423 push_dbg(D_FSM);
7321 IFDBG(D_NONE, FN; STRLIT("state x_start"););
7322 3423 empty_prop_input_queue();
7323 3423 empty_synode_number_pool();
7324 3423 reset_snapshot_mask();
7325 3423 set_last_received_config(null_synode);
7326
7327 3423 SET_X_FSM_STATE(xcom_fsm_start);
7328 3423 return 1;
7329 }
7330
7331 848 static int handle_fsm_net_boot(task_arg fsmargs, xcom_fsm_state *ctxt,
7332 int cont) {
7333 848 app_data *a = (app_data *)get_void_arg(fsmargs);
7334 848 install_node_group(a);
7335
1/2
✓ Branch 0 taken 848 times.
✗ Branch 1 not taken.
 848 if (is_member(get_site_def())) {
7336 848 empty_prop_input_queue();
7337 848 empty_synode_number_pool();
7338 {
7339
1/2
✓ Branch 0 taken 848 times.
✗ Branch 1 not taken.
 848 synode_no start = get_site_def()->start;
7340
1/2
✓ Branch 0 taken 848 times.
✗ Branch 1 not taken.
 848 if (start.msgno == 0) { /* May happen during initial boot */
7341 848 start.msgno = 1; /* Start with first xcom message */
7342 /* If msgno is 0, it means that this node installed a unified_boot
7343 which came from the client, thus this node is the one that will send
7344 the unified_boot on xcom, so set the node number of start accordingly
7345 */
7346
2/4
✓ Branch 0 taken 848 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 848 times.
✗ Branch 3 not taken.
 848 start.node = get_nodeno(get_site_def());
7347 }
7348
1/2
✓ Branch 0 taken 848 times.
✗ Branch 1 not taken.
 848 set_executed_msg(start);
7349 }
7350 848 pop_dbg();
7351 848 SET_X_FSM_STATE(xcom_fsm_run_enter);
7352 848 cont = 1;
7353 }
7354 848 return cont;
7355 }
7356
7357 1367 static int handle_fsm_snapshot(task_arg fsmargs, xcom_fsm_state *ctxt) {
7358 1367 gcs_snapshot *gcs = (gcs_snapshot *)get_void_arg(fsmargs);
7359 1367 empty_prop_input_queue();
7360 1367 empty_synode_number_pool();
7361 1367 set_log_end(gcs);
7362 1367 handle_x_snapshot(gcs);
7363
7364 /* Get recovery manager going again */
7365
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1367 times.
 1367 if (recovery_restart_cb) recovery_restart_cb();
7366
7367 /* If we run under control of the recovery manager, we need to call
7368 * recovery_begin_cb to rendezvous with the recovery manager */
7369
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1367 times.
 1367 if (recovery_begin_cb) recovery_begin_cb();
7370
7371 /* If we run under control of the recovery manager, we need to call
7372 * recovery_end_cb to rendezvous with the recovery manager */
7373
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1367 times.
 1367 if (recovery_end_cb) recovery_end_cb();
7374
7375 /* If we are here, it means that we are recovering from another node
7376 */
7377 /* Do not bother to wait for more snapshots, just handle it and
7378 enter run state */
7379 1367 pop_dbg();
7380 1367 SET_X_FSM_STATE(xcom_fsm_run_enter);
7381 1367 return 1;
7382 }
7383
7384 /* purecov: begin deadcode */
7385 ✗ static int handle_fsm_snapshot_wait(xcom_fsm_state *ctxt) {
7386 ✗ empty_prop_input_queue();
7387 ✗ empty_synode_number_pool();
7388 ✗ start_x_timer(SNAPSHOT_WAIT_TIME);
7389 ✗ pop_dbg();
7390 ✗ SET_X_FSM_STATE(xcom_fsm_snapshot_wait_enter);
7391 ✗ return 1;
7392 }
7393 /* purecov: end */
7394
7395 2378 static void handle_fsm_exit() {
7396 /* Xcom is finished when we get here */
7397 2378 push_dbg(D_BUG);
7398 2378 bury_site(get_group_id(get_site_def()));
7399 2378 task_terminate_all(); /* Kill, kill, kill, kill, kill, kill. This is
7400 the end. */
7401
7402 /* init_xcom_base(); */ /* Reset shared variables */
7403 2378 init_tasks(); /* Reset task variables */
7404 2378 free_site_defs();
7405 2378 free_forced_config_site_def();
7406 2378 wait_forced_config = 0;
7407 2378 garbage_collect_servers();
7408 IFDBG(D_NONE, FN; STRLIT("shutting down"));
7409 2378 xcom_shutdown = 1;
7410 2378 start_config = null_synode;
7411
2/2
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 2377 times.
 2378 G_DEBUG("Exiting xcom thread");
7412 2378 }
7413
7414 /* start state */
7415 9356 static int xcom_fsm_start(xcom_actions action, task_arg fsmargs,
7416 xcom_fsm_state *ctxt) {
7417 static int need_init_cache = 0;
7418 9356 int cont = 0; /* Set to 1 if we should continue execution */
7419
7420
5/6
✓ Branch 0 taken 2385 times.
✓ Branch 1 taken 848 times.
✓ Branch 2 taken 1367 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2378 times.
✓ Branch 5 taken 2378 times.
 9356 switch (action) {
7421 2385 case x_fsm_init:
7422 2385 xcom_shutdown = 0;
7423 2385 sent_alive = 0.0;
7424 2385 oom_abort = 0;
7425
2/2
✓ Branch 0 taken 1170 times.
✓ Branch 1 taken 1215 times.
 2385 if (need_init_cache) init_cache();
7426 2385 break;
7427
7428 848 case x_fsm_net_boot:
7429 848 cont = handle_fsm_net_boot(fsmargs, ctxt, cont);
7430 848 break;
7431
7432 1367 case x_fsm_snapshot:
7433 1367 cont = handle_fsm_snapshot(fsmargs, ctxt);
7434 1367 break;
7435
7436 /* This is the entry point for the initial recovery after the process
7437 * has started when running under an external recovery manager. */
7438 /* If we get x_fsm_snapshot_wait, we are called from the recovery
7439 * manager thread */
7440 /* purecov: begin deadcode */
7441 ✗ case x_fsm_snapshot_wait:
7442 ✗ cont = handle_fsm_snapshot_wait(ctxt);
7443 ✗ break;
7444 /* purecov: end */
7445
7446 2378 case x_fsm_exit:
7447 2378 handle_fsm_exit();
7448 2378 break;
7449
7450 2378 default:
7451 2378 break;
7452 }
7453 9356 need_init_cache = 1;
7454 9356 return cont;
7455 }
7456
7457 /* snapshot_wait_enter state */
7458 /* purecov: begin deadcode */
7459 ✗ static int xcom_fsm_snapshot_wait_enter(xcom_actions action, task_arg fsmargs,
7460 xcom_fsm_state *ctxt) {
7461 (void)action;
7462 (void)fsmargs;
7463 ✗ push_dbg(D_DETECT | D_FSM | D_FILEOP | D_CONS | D_BASE | D_TRANSPORT);
7464 IFDBG(D_NONE, FN; STRLIT("state x_snapshot_wait"););
7465 ✗ log_start_max = null_synode;
7466 ✗ log_end_max = null_synode;
7467 ✗ SET_X_FSM_STATE(xcom_fsm_snapshot_wait);
7468 ✗ return 0;
7469 }
7470 /* purecov: end */
7471
7472 /* purecov: begin deadcode */
7473 ✗ static int handle_local_snapshot(task_arg fsmargs, xcom_fsm_state *ctxt) {
7474 ✗ update_best_snapshot((gcs_snapshot *)get_void_arg(fsmargs));
7475 /* When recovering locally, fetch node number from site_def after
7476 * processing the snapshot */
7477 ✗ note_snapshot(get_site_def()->nodeno);
7478 ✗ send_need_boot();
7479 ✗ pop_dbg();
7480 ✗ SET_X_FSM_STATE(xcom_fsm_recover_wait_enter);
7481 ✗ return 1;
7482 }
7483 /* purecov: end */
7484
7485 /* purecov: begin deadcode */
7486 ✗ static int handle_snapshot(task_arg fsmargs, xcom_fsm_state *ctxt) {
7487 /* Snapshot from another node */
7488 ✗ gcs_snapshot *gcs = (gcs_snapshot *)get_void_arg(fsmargs);
7489 ✗ set_log_end(gcs);
7490 ✗ update_best_snapshot(gcs);
7491 /* We now have a site, so note that we have processed the local
7492 * snapshot even if we have not seen one, since if we are here, no
7493 * local snapshot will ever arrive. This simplifies the test in
7494 * got_all_snapshots() */
7495 ✗ note_snapshot(get_site_def()->nodeno);
7496 ✗ send_need_boot();
7497 ✗ pop_dbg();
7498 ✗ SET_X_FSM_STATE(xcom_fsm_recover_wait_enter);
7499 ✗ return 1;
7500 }
7501 /* purecov: end */
7502
7503 /* snapshot_wait state */
7504 /* purecov: begin deadcode */
7505 ✗ static int xcom_fsm_snapshot_wait(xcom_actions action, task_arg fsmargs,
7506 xcom_fsm_state *ctxt) {
7507 ✗ switch (action) {
7508 /* If we get x_fsm_local_snapshot, we are called from the recovery
7509 * manager thread */
7510 ✗ case x_fsm_local_snapshot:
7511 ✗ return handle_local_snapshot(fsmargs, ctxt);
7512
7513 ✗ case x_fsm_snapshot:
7514 ✗ return handle_snapshot(fsmargs, ctxt);
7515
7516 ✗ case x_fsm_timeout:
7517 /* Will time out if no snapshot available */
7518 /* If we run under control of the recovery manager, we need to call
7519 * recovery_end_cb to rendezvous with the recovery manager */
7520 ✗ if (recovery_end_cb) recovery_end_cb();
7521 ✗ pop_dbg();
7522 ✗ SET_X_FSM_STATE(xcom_fsm_start_enter);
7523 ✗ return 1;
7524
7525 ✗ default:
7526 ✗ break;
7527 }
7528 ✗ return 0;
7529 }
7530 /* purecov: end */
7531
7532 /* recover_wait_enter state */
7533 /* purecov: begin deadcode */
7534 ✗ static int xcom_fsm_recover_wait_enter(xcom_actions action, task_arg fsmargs,
7535 xcom_fsm_state *ctxt) {
7536 (void)action;
7537 (void)fsmargs;
7538 ✗ push_dbg(D_DETECT | D_FSM | D_FILEOP | D_CONS | D_BASE | D_TRANSPORT);
7539 IFDBG(D_NONE, FN; STRLIT("state x_recover_wait"););
7540 ✗ if (got_all_snapshots()) {
7541 /* Need to send message to trigger transition in context of xcom
7542 * thread */
7543 ✗ send_x_fsm_complete();
7544 }
7545 ✗ SET_X_FSM_STATE(xcom_fsm_recover_wait);
7546 ✗ return 0;
7547 }
7548 /* purecov: end */
7549
7550 /* recover_wait state */
7551 /* purecov: begin deadcode */
7552 ✗ static int xcom_fsm_recover_wait(xcom_actions action, task_arg fsmargs,
7553 xcom_fsm_state *ctxt) {
7554 ✗ if (action == x_fsm_snapshot) {
7555 ✗ gcs_snapshot *gcs = (gcs_snapshot *)get_void_arg(fsmargs);
7556 ✗ set_log_end(gcs);
7557 ✗ update_best_snapshot(gcs);
7558 ✗ } else if (action == x_fsm_timeout || action == x_fsm_complete) {
7559 /* Wait terminated by timeout or because all nodes have sent a
7560 * snapshot */
7561 /* If we run under control of the recovery manager, we need to call
7562 * recovery_end_cb to rendezvous with the recovery manager */
7563 ✗ if (recovery_end_cb) recovery_end_cb();
7564 ✗ pop_dbg();
7565 ✗ SET_X_FSM_STATE(xcom_fsm_run_enter);
7566 ✗ return 1;
7567 }
7568 ✗ if (got_all_snapshots()) {
7569 /* Need to send message to trigger transition in context of xcom
7570 * thread */
7571 ✗ send_x_fsm_complete();
7572 }
7573 ✗ return 0;
7574 }
7575 /* purecov: end */
7576
7577 /* run_enter state */
7578 2215 static int xcom_fsm_run_enter(xcom_actions action, task_arg fsmargs,
7579 xcom_fsm_state *ctxt) {
7580 (void)action;
7581 (void)fsmargs;
7582 2215 start_config = get_site_def()->boot_key;
7583
7584 /* Final sanity check of executed_msg */
7585
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2215 times.
 2215 if (find_site_def(executed_msg) == nullptr) {
7586 /* No site_def matches executed_msg, set it to site->start */
7587 ✗ set_executed_msg(get_site_def()->start);
7588 }
7589
7590 IFDBG(D_NONE, FN; STRLIT("state x_run"););
7591 IFDBG(D_BUG, FN; SYCEXP(executed_msg););
7592 IFDBG(D_BUG, FN; SYCEXP(start_config););
7593 2215 stop_x_timer();
7594
1/2
✓ Branch 0 taken 2215 times.
✗ Branch 1 not taken.
 2215 if (xcom_run_cb) xcom_run_cb(0);
7595 2215 client_boot_done = 1;
7596 2215 netboot_ok = 1;
7597 2215 set_proposer_startpoint();
7598 2215 create_proposers();
7599 2215 set_task(&executor, task_new(executor_task, null_arg, "executor_task",
7600 XCOM_THREAD_DEBUG));
7601 2215 set_task(&sweeper,
7602 task_new(sweeper_task, null_arg, "sweeper_task", XCOM_THREAD_DEBUG));
7603 2215 set_task(&detector, task_new(detector_task, null_arg, "detector_task",
7604 XCOM_THREAD_DEBUG));
7605 2215 set_task(&alive_t,
7606 task_new(alive_task, null_arg, "alive_task", XCOM_THREAD_DEBUG));
7607 2215 set_task(&cache_task, task_new(cache_manager_task, null_arg,
7608 "cache_manager_task", XCOM_THREAD_DEBUG));
7609
7610 2215 push_dbg(D_FSM /* | D_EXEC | D_BASE | D_TRANSPORT */);
7611 2215 SET_X_FSM_STATE(xcom_fsm_run);
7612 2215 return 1;
7613 }
7614
7615 2208 static int handle_fsm_terminate(task_arg fsmargs, xcom_fsm_state *ctxt) {
7616 2208 dump_debug_exec_state();
7617 2208 client_boot_done = 0;
7618 2208 netboot_ok = 0;
7619 2208 oom_abort = 0;
7620 2208 terminate_proposers();
7621 2208 init_proposers();
7622 2208 task_terminate(executor);
7623 2208 set_task(&executor, nullptr);
7624 2208 task_terminate(sweeper);
7625 2208 set_task(&sweeper, nullptr);
7626 2208 task_terminate(detector);
7627 2208 set_task(&detector, nullptr);
7628 2208 task_terminate(alive_t);
7629 2208 set_task(&alive_t, nullptr);
7630 2208 task_terminate(cache_task);
7631 2208 set_task(&cache_task, nullptr);
7632
7633 2208 init_xcom_base(); /* Reset shared variables */
7634 2208 free_site_defs();
7635 2208 free_forced_config_site_def();
7636 2208 wait_forced_config = 0;
7637 2208 garbage_collect_servers();
7638
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2208 times.
 2208 if (xcom_terminate_cb) xcom_terminate_cb(get_int_arg(fsmargs));
7639 2208 pop_dbg();
7640 2208 SET_X_FSM_STATE(xcom_fsm_start_enter);
7641 2208 return 1;
7642 }
7643
7644 27 static void handle_fsm_force_config(task_arg fsmargs) {
7645 27 app_data *a = (app_data *)get_void_arg(fsmargs);
7646 27 site_def *s = create_site_def_with_start(a, executed_msg);
7647
7648 27 s->boot_key = executed_msg;
7649 27 invalidate_servers(get_site_def(), s);
7650 27 start_force_config(s, 1);
7651 27 wait_forced_config = 1; /* Note that forced config has not yet arrived */
7652 27 }
7653
7654 /* run state */
7655 5298 static int xcom_fsm_run(xcom_actions action, task_arg fsmargs,
7656 xcom_fsm_state *ctxt) {
7657
3/4
✓ Branch 0 taken 2208 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 27 times.
✓ Branch 3 taken 3063 times.
 5298 switch (action) {
7658 2208 case x_fsm_terminate:
7659 2208 return handle_fsm_terminate(fsmargs, ctxt);
7660
7661 /* purecov: begin deadcode */
7662 ✗ case x_fsm_need_snapshot:
7663 IFDBG(D_NONE, STRLIT("got snapshot request in x_run state"));
7664 ✗ break;
7665 /* purecov: end */
7666
7667 27 case x_fsm_force_config:
7668 27 handle_fsm_force_config(fsmargs);
7669 27 break;
7670
7671 3063 default:
7672 3063 break;
7673 }
7674 3090 return 0;
7675 }
7676
7677 /* Trampoline which loops calling thunks pointed to by ctxt.state_fp until 0
7678 * is returned. Return pointer to ctxt. */
7679 10231 xcom_fsm_state *xcom_fsm_impl(xcom_actions action, task_arg fsmargs) {
7680 static xcom_fsm_state ctxt = X_FSM_STATE(xcom_fsm_init);
7681
7682
2/2
✓ Branch 0 taken 7 times.
✓ Branch 1 taken 10224 times.
 10231 G_DEBUG("%f pid %d xcom_id %x state %s action %s", seconds(), xpid(),
7683 get_my_xcom_id(), ctxt.state_name, xcom_actions_name[action]);
7684 ADD_DBG(D_FSM, add_event(EVENT_DUMP_PAD, string_arg("state"));
7685 add_event(EVENT_DUMP_PAD, string_arg(ctxt.state_name));
7686 add_event(EVENT_DUMP_PAD, string_arg("action"));
7687 add_event(EVENT_DUMP_PAD, string_arg(xcom_actions_name[action]));
7688 add_event(EVENT_DUMP_PAD, string_arg("executed_msg"));
7689 add_synode_event(executed_msg););
7690 #ifdef TASK_EVENT_TRACE
7691 dump_task_events();
7692 #endif
7693 /* Crank the state machine until it stops */
7694 IFDBG(D_BUG, FN; STREXP(ctxt.state_name); STREXP(xcom_actions_name[action]));
7695
2/2
✓ Branch 0 taken 11276 times.
✓ Branch 1 taken 10231 times.
 21507 while (ctxt.state_fp(action, fsmargs, &ctxt)) {
7696 IFDBG(D_BUG, FN; STREXP(ctxt.state_name);
7697 STREXP(xcom_actions_name[action]));
7698 }
7699 10231 return &ctxt;
7700 }
7701
7702 /* Call FSM trampoline and return state name of resulting state */
7703 10231 char const *xcom_fsm(xcom_actions action, task_arg fsmargs) {
7704 10231 xcom_fsm_state *s = xcom_fsm_impl(action, fsmargs);
7705 10231 return s->state_name;
7706 }
7707
7708 /* See if we can send a snapshot to another node */
7709 /* purecov: begin deadcode */
7710 ✗ static int can_send_snapshot() {
7711 ✗ xcom_fsm_state *state = xcom_fsm_impl(x_fsm_need_snapshot, null_arg);
7712 ✗ return state->state_fp == xcom_fsm_run;
7713 }
7714 /* purecov: end */
7715
7716 2292 void set_app_snap_handler(app_snap_handler x) { handle_app_snap_cb = x; }
7717
7718 /* purecov: begin deadcode */
7719 ✗ void set_app_snap_getter(app_snap_getter x) { get_app_snap_cb = x; }
7720 /* purecov: end */
7721
7722 /* Read max n bytes from socket fd into buffer buf */
7723 5888 static result socket_read(connection_descriptor *rfd, void *buf, int n) {
7724 5888 result ret = {0, 0};
7725
7726
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5888 times.
 5888 assert(n >= 0);
7727
7728 do {
7729
1/2
✓ Branch 0 taken 107182 times.
✗ Branch 1 not taken.
 107182 ret = con_read(rfd, buf, n);
7730 107182 task_dump_err(ret.funerr);
7731
5/6
✓ Branch 0 taken 101294 times.
✓ Branch 1 taken 5888 times.
✓ Branch 2 taken 101294 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 101294 times.
✓ Branch 5 taken 5888 times.
 107182 } while (ret.val < 0 && can_retry_read(ret.funerr));
7732 5888 return ret;
7733 }
7734
7735 /* Read exactly n bytes from socket fd into buffer buf */
7736 5888 static int64_t socket_read_bytes(connection_descriptor *rfd, char *p,
7737 uint32_t n) {
7738 5888 uint32_t left = n;
7739 5888 char *bytes = p;
7740
7741 5888 result nread = {0, 0};
7742
7743
2/2
✓ Branch 0 taken 5888 times.
✓ Branch 1 taken 5688 times.
 11576 while (left > 0) {
7744 /*
7745 socket_read just reads no more than INT_MAX bytes. We should not pass
7746 a length more than INT_MAX to it.
7747 */
7748 5888 int r = (int)MIN(left, INT_MAX);
7749
7750
1/2
✓ Branch 0 taken 5888 times.
✗ Branch 1 not taken.
 5888 nread = socket_read(rfd, bytes, r);
7751
2/2
✓ Branch 0 taken 200 times.
✓ Branch 1 taken 5688 times.
 5888 if (nread.val == 0) {
7752 200 return 0;
7753
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5688 times.
 5688 } else if (nread.val < 0) {
7754 ✗ return -1;
7755 } else {
7756 5688 bytes += nread.val;
7757 5688 left -= (uint32_t)nread.val;
7758 }
7759 }
7760
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 5688 times.
 5688 assert(left == 0);
7761 5688 return n;
7762 }
7763
7764 /* Write n bytes from buffer buf to socket fd */
7765 83584 static int64_t socket_write(connection_descriptor *wfd, void *_buf, uint32_t n,
7766 connnection_write_method write_function) {
7767 83584 char *buf = (char *)_buf;
7768 83584 result ret = {0, 0};
7769
7770 uint32_t total; /* Keeps track of number of bytes written so far */
7771
7772 83584 total = 0;
7773
2/2
✓ Branch 0 taken 83585 times.
✓ Branch 1 taken 83320 times.
 166905 while (total < n) {
7774 83585 int w = (int)MIN(n - total, INT_MAX);
7775
7776
5/8
✓ Branch 0 taken 83586 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 265 times.
✓ Branch 3 taken 83321 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 265 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 83586 times.
 83850 while ((ret = write_function(wfd, buf + total, w)).val < 0 &&
7777 265 can_retry_write(ret.funerr)) {
7778 ✗ task_dump_err(ret.funerr);
7779 IFDBG(D_NONE, FN; STRLIT("retry "); NEXP(total, d); NEXP(n, d));
7780 }
7781
2/2
✓ Branch 0 taken 265 times.
✓ Branch 1 taken 83321 times.
 83586 if (ret.val <= 0) { /* Something went wrong */
7782 265 task_dump_err(ret.funerr);
7783 265 return -1;
7784 } else {
7785 83321 total += (uint32_t)ret.val; /* Add number of bytes written to total */
7786 }
7787 }
7788 IFDBG(D_TRANSPORT, FN; NEXP(total, u); NEXP(n, u));
7789
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 83320 times.
 83320 assert(total == n);
7790 83320 return (total);
7791 }
7792
7793 #define CONNECT_FAIL \
7794 ret_fd = -1; \
7795 goto end
7796
7797 ✗ connection_descriptor *xcom_open_client_connection(char const *server,
7798 xcom_port port) {
7799 ✗ return open_new_connection(server, port);
7800 }
7801
7802 /* Send a protocol negotiation message on connection con */
7803 2096 static int xcom_send_proto(connection_descriptor *con, xcom_proto x_proto,
7804 x_msg_type x_type, unsigned int tag) {
7805 char buf[MSG_HDR_SIZE];
7806 2096 memset(buf, 0, MSG_HDR_SIZE);
7807
7808
1/2
✓ Branch 0 taken 2096 times.
✗ Branch 1 not taken.
 2096 if (con->fd >= 0) {
7809 2096 con->snd_tag = tag;
7810
1/2
✓ Branch 0 taken 2096 times.
✗ Branch 1 not taken.
 2096 write_protoversion(VERS_PTR((unsigned char *)buf), x_proto);
7811
1/2
✓ Branch 0 taken 2096 times.
✗ Branch 1 not taken.
 2096 put_header_1_0((unsigned char *)buf, 0, x_type, tag);
7812 {
7813 int sent;
7814
1/2
✓ Branch 0 taken 2096 times.
✗ Branch 1 not taken.
 2096 sent = (int)socket_write(con, buf, MSG_HDR_SIZE);
7815
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2096 times.
 2096 if (con->fd < 0) {
7816 ✗ return -1;
7817 }
7818 2096 return sent;
7819 }
7820 } else {
7821 ✗ return -1;
7822 }
7823 }
7824
7825 2096 static int xcom_recv_proto(connection_descriptor *rfd, xcom_proto *x_proto,
7826 x_msg_type *x_type, unsigned int *tag) {
7827 int n;
7828 unsigned char header_buf[MSG_HDR_SIZE];
7829 uint32_t msgsize;
7830
7831 /* Read length field, protocol version, and checksum */
7832
1/2
✓ Branch 0 taken 2096 times.
✗ Branch 1 not taken.
 2096 n = (int)socket_read_bytes(rfd, (char *)header_buf, MSG_HDR_SIZE);
7833
7834
2/2
✓ Branch 0 taken 200 times.
✓ Branch 1 taken 1896 times.
 2096 if (n != MSG_HDR_SIZE) {
7835 IFDBG(D_NONE, FN; NDBG(n, d));
7836 200 return -1;
7837 }
7838
7839
1/2
✓ Branch 0 taken 1896 times.
✗ Branch 1 not taken.
 1896 *x_proto = read_protoversion(VERS_PTR(header_buf));
7840
1/2
✓ Branch 0 taken 1896 times.
✗ Branch 1 not taken.
 1896 get_header_1_0(header_buf, &msgsize, x_type, tag);
7841
7842 1896 return n;
7843 }
7844
7845 enum { TAG_START = 313 };
7846
7847 /**
7848 * @brief Checks if a given app_data is from a given cargo_type.
7849 *
7850 * @param a the app_data
7851 * @param t the cargo type
7852 * @return int TRUE (1) if app_data a is from cargo_type t
7853 */
7854
7855 1900 static inline int is_cargo_type(app_data_ptr a, cargo_type t) {
7856
3/4
✓ Branch 0 taken 1900 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1873 times.
✓ Branch 3 taken 27 times.
 1900 return a ? (a->body.c_t == t) : 0;
7857 }
7858
7859 /**
7860 * @brief Retrieves the address that was used in the add_node request
7861 *
7862 * @param a app data containing the node to add
7863 * @param member address we used to present ourselves to other nodes
7864 * @return char* a pointer to the address being added.
7865 */
7866 4 static char *get_add_node_address(app_data_ptr a, unsigned int *member) {
7867 4 char *retval = nullptr;
7868
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
 4 if (!is_cargo_type(a, add_node_type)) return nullptr;
7869
7870
2/2
✓ Branch 0 taken 3 times.
✓ Branch 1 taken 1 times.
 4 if ((*member) < a->body.app_u_u.nodes.node_list_len) {
7871 3 retval = a->body.app_u_u.nodes.node_list_val[(*member)].address;
7872 3 (*member)++;
7873 }
7874
7875 4 return retval;
7876 }
7877
7878 10455 int is_node_v4_reachable_with_info(struct addrinfo *retrieved_addr_info) {
7879 10455 int v4_reachable = 0;
7880
7881 /* Verify if we are reachable either by V4 and by V6 with the provided
7882 address. */
7883 10455 struct addrinfo *my_own_information_loop = nullptr;
7884
7885 10455 my_own_information_loop = retrieved_addr_info;
7886
4/4
✓ Branch 0 taken 10460 times.
✓ Branch 1 taken 10450 times.
✓ Branch 2 taken 10455 times.
✓ Branch 3 taken 5 times.
 20910 while (!v4_reachable && my_own_information_loop) {
7887
2/2
✓ Branch 0 taken 10450 times.
✓ Branch 1 taken 5 times.
 10455 if (my_own_information_loop->ai_family == AF_INET) {
7888 10450 v4_reachable = 1;
7889 }
7890 10455 my_own_information_loop = my_own_information_loop->ai_next;
7891 }
7892
7893 10455 return v4_reachable;
7894 }
7895
7896 2 int is_node_v4_reachable(char *node_address) {
7897 2 int v4_reachable = 0;
7898
7899 /* Verify if we are reachable either by V4 and by V6 with the provided
7900 address. */
7901 2 struct addrinfo *my_own_information = nullptr;
7902
7903
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 checked_getaddrinfo(node_address, nullptr, nullptr, &my_own_information);
7904
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
 2 if (my_own_information == nullptr) {
7905 ✗ return v4_reachable;
7906 }
7907
7908
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 v4_reachable = is_node_v4_reachable_with_info(my_own_information);
7909
7910
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 if (my_own_information) freeaddrinfo(my_own_information);
7911
7912 2 return v4_reachable;
7913 }
7914
7915 3 int are_we_allowed_to_upgrade_to_v6(app_data_ptr a) {
7916 /* This should the address we used to present ourselves to other nodes. */
7917 3 unsigned int list_member = 0;
7918 3 char *added_node = nullptr;
7919
7920 3 int is_v4_reachable = 0;
7921
2/2
✓ Branch 0 taken 3 times.
✓ Branch 1 taken 1 times.
 4 while ((added_node = get_add_node_address(a, &list_member)) != nullptr) {
7922 xcom_port my_own_port;
7923 char my_own_address[IP_MAX_SIZE];
7924 int ip_and_port_error =
7925
1/2
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
 3 get_ip_and_port(added_node, my_own_address, &my_own_port);
7926
7927
2/2
✓ Branch 0 taken 1 times.
✓ Branch 1 taken 2 times.
 3 if (ip_and_port_error) {
7928
3/6
✓ Branch 0 taken 1 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1 times.
✗ Branch 5 not taken.
 1 G_DEBUG("Error retrieving IP and Port information");
7929 2 return 0;
7930 }
7931
7932 /* Verify if we are reachable either by V4 and by V6 with the provided
7933 address.
7934 This means that the other side won't be able to contact us since we
7935 do not provide a public V4 address */
7936
3/4
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 times.
✓ Branch 3 taken 1 times.
 2 if (!(is_v4_reachable = is_node_v4_reachable(my_own_address))) {
7937
2/4
✓ Branch 0 taken 1 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 times.
✗ Branch 3 not taken.
 1 G_ERROR(
7938 "Unable to add node to a group of older nodes. Please "
7939 "reconfigure "
7940 "you local address to an IPv4 address or configure your DNS to "
7941 "provide "
7942 "an IPv4 address");
7943 1 return 0;
7944 }
7945 }
7946
7947 1 return is_v4_reachable;
7948 }
7949
7950 2097 int64_t xcom_send_client_app_data(connection_descriptor *fd, app_data_ptr a,
7951 int force) {
7952
1/2
✓ Branch 0 taken 2097 times.
✗ Branch 1 not taken.
 2097 pax_msg *msg = pax_msg_new(null_synode, nullptr);
7953 2097 uint32_t buflen = 0;
7954 2097 char *buf = nullptr;
7955 2097 int64_t retval = 0;
7956 2097 int serialized = 0;
7957
7958
2/2
✓ Branch 0 taken 2096 times.
✓ Branch 1 taken 1 times.
 2097 if (!proto_done(fd)) {
7959 xcom_proto x_proto;
7960 x_msg_type x_type;
7961 unsigned int tag;
7962
1/2
✓ Branch 0 taken 2096 times.
✗ Branch 1 not taken.
 2096 retval = xcom_send_proto(fd, my_xcom_version, x_version_req, TAG_START);
7963
4/6
✓ Branch 0 taken 2096 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 times.
✓ Branch 3 taken 2095 times.
✓ Branch 4 taken 1 times.
✗ Branch 5 not taken.
 2096 G_DEBUG("client sent negotiation request for protocol %d", my_xcom_version);
7964
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2096 times.
 2296 if (retval < 0) goto end;
7965
1/2
✓ Branch 0 taken 2096 times.
✗ Branch 1 not taken.
 2096 retval = xcom_recv_proto(fd, &x_proto, &x_type, &tag);
7966
2/2
✓ Branch 0 taken 200 times.
✓ Branch 1 taken 1896 times.
 2096 if (retval < 0) goto end;
7967
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1896 times.
 1896 if (tag != TAG_START) {
7968 ✗ retval = -1;
7969 ✗ goto end;
7970 }
7971
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1896 times.
 1896 if (x_type != x_version_reply) {
7972 ✗ retval = -1;
7973 ✗ goto end;
7974 }
7975
7976
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1896 times.
 1896 if (x_proto == x_unknown_proto) {
7977 ✗ G_DEBUG("no common protocol, returning error");
7978 ✗ retval = -1;
7979 ✗ goto end;
7980 }
7981
7982 /* This code will check if, in case of an upgrade if:
7983 - We are a node able to speak IPv6.
7984 - If we are connecting to a group that does not speak IPv6.
7985 - If our address is IPv4-compatible in order for the old group to be
7986 able to contact us back. */
7987
5/8
✓ Branch 0 taken 1869 times.
✓ Branch 1 taken 27 times.
✓ Branch 2 taken 1869 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 1869 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 1896 times.
 1896 if (is_cargo_type(a, add_node_type) && x_proto < minimum_ipv6_version() &&
7988 ✗ !are_we_allowed_to_upgrade_to_v6(a)) {
7989 ✗ retval = -1;
7990 ✗ goto end;
7991 }
7992
7993
4/6
✓ Branch 0 taken 1896 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 times.
✓ Branch 3 taken 1895 times.
✓ Branch 4 taken 1 times.
✗ Branch 5 not taken.
 1896 G_DEBUG("client connection will use protocol version %d", x_proto);
7994 IFDBG(D_NONE, STRLIT("client connection will use protocol version ");
7995 NDBG(x_proto, u); STRLIT(xcom_proto_to_str(x_proto)));
7996 1896 fd->x_proto = x_proto;
7997 1896 set_connected(fd, CON_PROTO);
7998 }
7999 1897 msg->a = a;
8000 1897 msg->to = VOID_NODE_NO;
8001 1897 msg->op = client_msg;
8002 1897 msg->force_delivery = force;
8003
8004
1/2
✓ Branch 0 taken 1897 times.
✗ Branch 1 not taken.
 1897 serialized = serialize_msg(msg, fd->x_proto, &buflen, &buf);
8005
2/2
✓ Branch 0 taken 1896 times.
✓ Branch 1 taken 1 times.
 1897 if (serialized) {
8006
1/2
✓ Branch 0 taken 1896 times.
✗ Branch 1 not taken.
 1896 retval = socket_write(fd, buf, buflen);
8007 1896 if (buflen != retval) {
8008 IFDBG(D_NONE, FN; STRLIT("write failed "); NDBG(fd->fd, d);
8009 NDBG(buflen, d); NDBG64(retval));
8010 }
8011 } else {
8012 /* Failed to serialize, set retval accordingly. */
8013 1 retval = -1;
8014 }
8015 1897 X_FREE(buf);
8016 2097 end:
8017 2097 msg->a = nullptr; /* Do not deallocate a */
8018
1/2
✓ Branch 0 taken 2097 times.
✗ Branch 1 not taken.
 2097 XCOM_XDR_FREE(xdr_pax_msg, msg);
8019 2097 return retval;
8020 }
8021
8022 /* purecov: begin tested */
8023 /*
8024 * Tested by TEST_F(XComMultinodeSmokeTest,
8025 * 3_nodes_member_crashes_with_dieop_and_joins_again_immediately) GCS smoke
8026 * test
8027 */
8028 ✗ int64_t xcom_client_send_die(connection_descriptor *fd) {
8029 ✗ if (!fd) return 0;
8030 ✗ uint32_t buflen = 0;
8031 ✗ char *buf = nullptr;
8032 ✗ int64_t retval = 0;
8033 app_data a;
8034 ✗ pax_msg *msg = pax_msg_new(null_synode, nullptr);
8035
8036 ✗ if (!proto_done(fd)) {
8037 xcom_proto x_proto;
8038 x_msg_type x_type;
8039 unsigned int tag;
8040 ✗ retval = xcom_send_proto(fd, my_xcom_version, x_version_req, TAG_START);
8041 ✗ G_DEBUG("client sent negotiation request for protocol %d", my_xcom_version);
8042 ✗ if (retval < 0) goto end;
8043 ✗ retval = xcom_recv_proto(fd, &x_proto, &x_type, &tag);
8044 ✗ if (retval < 0) goto end;
8045 ✗ if (tag != TAG_START) {
8046 ✗ retval = -1;
8047 ✗ goto end;
8048 }
8049 ✗ if (x_type != x_version_reply) {
8050 ✗ retval = -1;
8051 ✗ goto end;
8052 }
8053
8054 ✗ if (x_proto == x_unknown_proto) {
8055 ✗ G_DEBUG("no common protocol, returning error");
8056 ✗ retval = -1;
8057 ✗ goto end;
8058 }
8059 ✗ G_DEBUG("client connection will use protocol version %d", x_proto);
8060 IFDBG(D_NONE, STRLIT("client connection will use protocol version ");
8061 NDBG(x_proto, u); STRLIT(xcom_proto_to_str(x_proto)));
8062 ✗ fd->x_proto = x_proto;
8063 ✗ set_connected(fd, CON_PROTO);
8064 }
8065 ✗ init_app_data(&a);
8066 ✗ a.body.c_t = app_type;
8067 ✗ msg->a = &a;
8068 ✗ msg->op = die_op;
8069 /*
8070 Set the msgno to a value that ensures the die_op will be processed by
8071 XCom when it is received (it needs to be higher than the msgno of the
8072 executed_msg, otherwise XCom will simply ignore it).
8073 */
8074 ✗ msg->synode.msgno = UINT64_MAX;
8075
8076 ✗ serialize_msg(msg, fd->x_proto, &buflen, &buf);
8077 ✗ if (buflen) {
8078 ✗ retval = socket_write(fd, buf, buflen);
8079 ✗ if (buflen != retval) {
8080 IFDBG(D_NONE, FN; STRLIT("write failed "); NDBG(fd->fd, d);
8081 NDBG(buflen, d); NDBG64(retval));
8082 }
8083 ✗ X_FREE(buf);
8084 }
8085 ✗ xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8086 ✗ end:
8087 ✗ msg->a = nullptr;
8088 ✗ XCOM_XDR_FREE(xdr_pax_msg, msg);
8089 ✗ return retval > 0 && retval == buflen ? 1 : 0;
8090 }
8091 /* purecov: end */
8092
8093 #ifdef XCOM_STANDALONE
8094 /* purecov: begin deadcode */
8095 int64_t xcom_client_send_data(uint32_t size, char *data,
8096 connection_descriptor *fd) {
8097 if (!fd) return 0;
8098 app_data a;
8099 int64_t retval = 0;
8100 init_app_data(&a);
8101 a.body.c_t = app_type;
8102 a.body.app_u_u.data.data_len = size;
8103 a.body.app_u_u.data.data_val = data;
8104 retval = xcom_send_client_app_data(fd, &a, 0);
8105 xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8106 return retval;
8107 }
8108
8109 int64_t xcom_client_send_data_no_free(uint32_t size, char *data,
8110 connection_descriptor *fd) {
8111 if (!fd) return 0;
8112 app_data a;
8113 int64_t retval = 0;
8114 init_app_data(&a);
8115 a.body.c_t = app_type;
8116 a.body.app_u_u.data.data_len = size;
8117 a.body.app_u_u.data.data_val = data;
8118 retval = xcom_send_client_app_data(fd, &a, 0);
8119 return retval;
8120 }
8121 /* purecov: end */
8122 #endif
8123
8124 #ifndef _WIN32
8125 #include <arpa/inet.h>
8126 #include <netinet/in.h>
8127 #include <sys/socket.h>
8128 #endif
8129
8130 /* Output warning in log periodically if we receive messages
8131 with a protocol version that does not match our own */
8132 /* purecov: begin inspected */
8133 ✗ void warn_protoversion_mismatch(connection_descriptor *rfd) {
8134 struct sockaddr_storage sock_addr;
8135 ✗ socklen_t sock_size = sizeof(sock_addr);
8136
8137 ✗ if (task_now() - protoversion_warning_time > PROTOVERSION_WARNING_TIMEOUT) {
8138 ✗ if (0 ==
8139 ✗ xcom_getpeername(rfd->fd, (struct sockaddr *)&sock_addr, &sock_size)) {
8140 char buf[INET6_ADDRSTRLEN + 1];
8141 ✗ struct sockaddr_in *s4 = (struct sockaddr_in *)&sock_addr;
8142 ✗ struct sockaddr_in6 *s6 = (struct sockaddr_in6 *)&sock_addr;
8143 char const *ok;
8144
8145 ✗ memset((void *)buf, 0, sizeof(buf));
8146 ✗ if (sock_addr.ss_family == AF_INET) {
8147 ✗ ok = inet_ntop(sock_addr.ss_family, (void *)&s4->sin_addr, buf,
8148 sizeof(buf));
8149 } else {
8150 ✗ ok = inet_ntop(sock_addr.ss_family, (void *)&s6->sin6_addr, buf,
8151 sizeof(buf));
8152 }
8153 ✗ if (ok) {
8154 ✗ G_WARNING(
8155 "Detected incorrect xcom protocol version in connection from %s "
8156 "indicates missing cleanup of, or incorrect, xcom group "
8157 "definition on remote host. Please upgrade the process running on "
8158 "%s to a compatible version or stop it.",
8159 buf, buf);
8160 ✗ protoversion_warning_time = task_now();
8161 }
8162 }
8163 }
8164 }
8165 /* purecov: end */
8166
8167 1896 static pax_msg *socket_read_msg(connection_descriptor *rfd, pax_msg *p)
8168 /* Should buffer reads as well */
8169 {
8170 int64_t n;
8171 char *bytes;
8172 unsigned char header_buf[MSG_HDR_SIZE];
8173 xcom_proto x_version;
8174 uint32_t msgsize;
8175 x_msg_type x_type;
8176 unsigned int tag;
8177 1896 int deserialize_ok = 0;
8178
8179 1896 bytes = nullptr;
8180
8181 /* Read version, length, type, and tag */
8182
1/2
✓ Branch 0 taken 1896 times.
✗ Branch 1 not taken.
 1896 n = socket_read_bytes(rfd, (char *)header_buf, MSG_HDR_SIZE);
8183
8184
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1896 times.
 1896 if (n <= 0) {
8185 IFDBG(D_NONE, FN; NDBG64(n));
8186 ✗ return nullptr;
8187 }
8188
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1896 times.
 1896 assert(n == MSG_HDR_SIZE);
8189 1896 x_version = (xcom_proto)get_32(VERS_PTR(header_buf));
8190 /* Check the protocol version before doing anything else */
8191 #ifdef XCOM_PARANOID
8192 assert(check_protoversion(x_version, rfd->x_proto));
8193 #endif
8194
2/4
✓ Branch 0 taken 1896 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 1896 times.
 1896 if (!check_protoversion(x_version, rfd->x_proto)) {
8195 /* purecov: begin inspected */
8196 ✗ warn_protoversion_mismatch(rfd);
8197 ✗ return nullptr;
8198 /* purecov: end */
8199 }
8200
8201 /* OK, we can grok this version */
8202
8203
1/2
✓ Branch 0 taken 1896 times.
✗ Branch 1 not taken.
 1896 get_header_1_0(header_buf, &msgsize, &x_type, &tag);
8204
8205 /* Allocate buffer space for message */
8206 1896 bytes = (char *)xcom_calloc(1, msgsize);
8207
8208 /* Read message */
8209
1/2
✓ Branch 0 taken 1896 times.
✗ Branch 1 not taken.
 1896 n = socket_read_bytes(rfd, bytes, msgsize);
8210
8211
1/2
✓ Branch 0 taken 1896 times.
✗ Branch 1 not taken.
 1896 if (n > 0) {
8212 /* Deserialize message */
8213
1/2
✓ Branch 0 taken 1896 times.
✗ Branch 1 not taken.
 1896 deserialize_ok = deserialize_msg(p, rfd->x_proto, bytes, msgsize);
8214 IFDBG(D_NONE, FN; STRLIT(" deserialized message"));
8215 }
8216 /* Deallocate buffer */
8217 1896 X_FREE(bytes);
8218
2/4
✓ Branch 0 taken 1896 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 1896 times.
 1896 if (n <= 0 || deserialize_ok == 0) {
8219 IFDBG(D_NONE, FN; NDBG64(n));
8220 ✗ return nullptr;
8221 }
8222 1896 return (p);
8223 }
8224
8225 #ifdef XCOM_STANDALONE
8226 /* purecov: begin deadcode */
8227 int xcom_client_boot(connection_descriptor *fd, node_list *nl,
8228 uint32_t group_id) {
8229 if (!fd) return 0;
8230 app_data a;
8231 int retval = 0;
8232 retval = (int)xcom_send_client_app_data(
8233 fd, init_config_with_group(&a, nl, unified_boot_type, group_id), 0);
8234 xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8235 return retval;
8236 }
8237 /* purecov: end */
8238 #endif
8239
8240 enum xcom_send_app_wait_result {
8241 SEND_REQUEST_FAILED = 0,
8242 RECEIVE_REQUEST_FAILED,
8243 REQUEST_BOTCHED,
8244 RETRIES_EXCEEDED,
8245 REQUEST_OK_RECEIVED,
8246 REQUEST_FAIL_RECEIVED,
8247 REQUEST_OK_REDIRECT
8248 };
8249 typedef enum xcom_send_app_wait_result xcom_send_app_wait_result;
8250
8251 /**
8252 * Send a message and wait for response.
8253 *
8254 * The caller is reponsible for freeing p after calling this function,
8255 * i.e. xdr_free((xdrproc_t)xdr_pax_msg, (char *)p)
8256 */
8257 2097 static xcom_send_app_wait_result xcom_send_app_wait_and_get(
8258 connection_descriptor *fd, app_data *a, int force, pax_msg *p,
8259 leader_info_data *leaders) {
8260 2097 int retval = 0;
8261 2097 int retry_count = 10; /* Same as 'connection_attempts' */
8262 2097 pax_msg *rp = nullptr;
8263
8264 do {
8265 2097 retval = (int)xcom_send_client_app_data(fd, a, force);
8266 2097 memset(p, 0, sizeof(*p)); /* before return so caller can free p */
8267
2/2
✓ Branch 0 taken 201 times.
✓ Branch 1 taken 1896 times.
 2097 if (retval < 0) {
8268 201 return SEND_REQUEST_FAILED;
8269 }
8270 1896 rp = socket_read_msg(fd, p);
8271
1/2
✓ Branch 0 taken 1896 times.
✗ Branch 1 not taken.
 1896 if (rp) {
8272 1896 client_reply_code cli_err = rp->cli_err;
8273
2/5
✓ Branch 0 taken 1367 times.
✓ Branch 1 taken 529 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 1896 switch (cli_err) {
8274 1367 case REQUEST_OK:
8275 1367 return REQUEST_OK_RECEIVED;
8276 529 case REQUEST_FAIL:
8277
2/4
✓ Branch 0 taken 529 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 529 times.
✗ Branch 3 not taken.
 529 G_INFO(
8278 "Sending a request to a remote XCom failed. Please check the "
8279 "remote node log for more details.")
8280 529 return REQUEST_FAIL_RECEIVED;
8281 ✗ case REQUEST_RETRY:
8282 ✗ if (retry_count > 1) xdr_free((xdrproc_t)xdr_pax_msg, (char *)p);
8283 ✗ G_INFO(
8284 "Retrying a request to a remote XCom. Please check the remote "
8285 "node log for more details.")
8286 ✗ xcom_sleep(1);
8287 ✗ break;
8288 ✗ case REQUEST_REDIRECT:
8289 ✗ G_DEBUG("cli_err %d", cli_err);
8290 ✗ if (leaders && rp->rd && rp->rd->rt == leader_info) {
8291 ✗ *leaders = steal_leader_info_data(rp->rd->reply_data_u.leaders);
8292 }
8293 ✗ xdr_free((xdrproc_t)xdr_pax_msg, (char *)p);
8294 ✗ return REQUEST_OK_REDIRECT;
8295 ✗ default:
8296 ✗ G_WARNING("XCom client connection has received an unknown response.");
8297 ✗ return REQUEST_BOTCHED;
8298 }
8299 } else {
8300 ✗ G_WARNING("Reading a request from a remote XCom failed.");
8301 ✗ return RECEIVE_REQUEST_FAILED;
8302 }
8303 ✗ } while (--retry_count);
8304 /* Timeout after REQUEST_RETRY has been received 'retry_count' times */
8305 ✗ G_MESSAGE(
8306 "Request failed: maximum number of retries (10) has been exhausted.");
8307 ✗ return RETRIES_EXCEEDED;
8308 }
8309
8310 2096 static int xcom_send_app_wait(connection_descriptor *fd, app_data *a, int force,
8311 leader_info_data *leaders) {
8312 pax_msg p;
8313 2096 int result = 0;
8314 2096 memset(&p, 0, sizeof(p));
8315 xcom_send_app_wait_result res =
8316
1/2
✓ Branch 0 taken 2096 times.
✗ Branch 1 not taken.
 2096 xcom_send_app_wait_and_get(fd, a, force, &p, leaders);
8317
2/3
✓ Branch 0 taken 729 times.
✓ Branch 1 taken 1367 times.
✗ Branch 2 not taken.
 2096 switch (res) {
8318 729 case SEND_REQUEST_FAILED:
8319 case RECEIVE_REQUEST_FAILED:
8320 case REQUEST_BOTCHED:
8321 case RETRIES_EXCEEDED:
8322 case REQUEST_FAIL_RECEIVED:
8323 case REQUEST_OK_REDIRECT:
8324 729 result = 0;
8325 729 break;
8326 1367 case REQUEST_OK_RECEIVED:
8327 1367 result = 1;
8328 1367 break;
8329 }
8330
1/2
✓ Branch 0 taken 2096 times.
✗ Branch 1 not taken.
 2096 xdr_free((xdrproc_t)xdr_pax_msg, (char *)&p);
8331 2096 return result;
8332 }
8333
8334 2096 int xcom_send_cfg_wait(connection_descriptor *fd, node_list *nl,
8335 uint32_t group_id, cargo_type ct, int force) {
8336 app_data a;
8337 2096 int retval = 0;
8338 IFDBG(D_NONE, FN; COPY_AND_FREE_GOUT(dbg_list(nl)););
8339
2/4
✓ Branch 0 taken 2096 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2096 times.
✗ Branch 3 not taken.
 2096 retval = xcom_send_app_wait(fd, init_config_with_group(&a, nl, ct, group_id),
8340 force, nullptr);
8341
1/2
✓ Branch 0 taken 2096 times.
✗ Branch 1 not taken.
 2096 xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8342 2096 return retval;
8343 }
8344
8345 2069 int xcom_client_add_node(connection_descriptor *fd, node_list *nl,
8346 uint32_t group_id) {
8347
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2069 times.
 2069 if (!fd) return 0;
8348 u_int i;
8349
2/2
✓ Branch 0 taken 2069 times.
✓ Branch 1 taken 2069 times.
 4138 for (i = 0; i < nl->node_list_len; i++) {
8350
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2069 times.
 2069 assert(nl->node_list_val[i].proto.max_proto > x_unknown_proto);
8351 }
8352 2069 return xcom_send_cfg_wait(fd, nl, group_id, add_node_type, 0);
8353 }
8354
8355 27 int xcom_client_remove_node(connection_descriptor *fd, node_list *nl,
8356 uint32_t group_id) {
8357
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 27 times.
 27 if (!fd) return 0;
8358 27 return xcom_send_cfg_wait(fd, nl, group_id, remove_node_type, 0);
8359 }
8360
8361 ✗ static int xcom_check_reply(int const res) {
8362 ✗ return res == REQUEST_OK_RECEIVED;
8363 }
8364
8365 #if 0
8366 /* purecov: begin deadcode */
8367 int xcom_client_get_event_horizon(connection_descriptor *fd, uint32_t group_id,
8368 xcom_event_horizon *event_horizon) {
8369 if (!fd) return 0;
8370 pax_msg p;
8371 app_data a;
8372 int result = 0;
8373
8374 memset(&p, 0, sizeof(p));
8375
8376 xcom_send_app_wait_result res = xcom_send_app_wait_and_get(
8377 fd, init_get_event_horizon_msg(&a, group_id), 0, &p, 0);
8378 result = xcom_check_reply(res);
8379 if (result) {
8380 *event_horizon = p.event_horizon;
8381 }
8382
8383 xdr_free((xdrproc_t)xdr_pax_msg, (char *)&p);
8384 xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8385
8386 return result;
8387 }
8388 /* purecov: end */
8389
8390 /* purecov: begin deadcode */
8391 int xcom_client_set_event_horizon(connection_descriptor *fd, uint32_t group_id,
8392 xcom_event_horizon event_horizon) {
8393 if (!fd) return 0;
8394 app_data a;
8395 int retval = 0;
8396 retval = xcom_send_app_wait(
8397 fd, init_set_event_horizon_msg(&a, group_id, event_horizon), 0, 0);
8398 xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8399 return retval;
8400 }
8401 /* purecov: end */
8402 #endif
8403
8404 1 int xcom_client_get_synode_app_data(connection_descriptor *const fd,
8405 uint32_t group_id,
8406 synode_no_array *const synodes,
8407 synode_app_data_array *const reply) {
8408
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 times.
 1 if (!fd) return 0;
8409 1 bool_t const success = TRUE;
8410 1 bool_t const failure = FALSE;
8411 1 bool_t result = failure;
8412 pax_msg p;
8413 app_data a;
8414 1 u_int const nr_synodes_requested = synodes->synode_no_array_len;
8415
8416 /* This call moves, as in C++ move semantics, synodes into app_data a. */
8417
1/2
✓ Branch 0 taken 1 times.
✗ Branch 1 not taken.
 1 init_get_synode_app_data_msg(&a, group_id, synodes);
8418
8419 {
8420 xcom_send_app_wait_result res =
8421
1/2
✓ Branch 0 taken 1 times.
✗ Branch 1 not taken.
 1 xcom_send_app_wait_and_get(fd, &a, 0, &p, nullptr);
8422
1/3
✓ Branch 0 taken 1 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
 1 switch (res) {
8423 1 case RECEIVE_REQUEST_FAILED:
8424 case REQUEST_BOTCHED:
8425 case RETRIES_EXCEEDED:
8426 case SEND_REQUEST_FAILED:
8427 case REQUEST_FAIL_RECEIVED:
8428 case REQUEST_OK_REDIRECT: {
8429
3/6
✓ Branch 0 taken 1 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 1 times.
✗ Branch 5 not taken.
 1 G_TRACE(
8430 "xcom_client_get_synode_app_data: XCom did not have the required "
8431 "%u "
8432 "synodes.",
8433 nr_synodes_requested);
8434 1 break;
8435 }
8436 ✗ case REQUEST_OK_RECEIVED: {
8437 ✗ u_int const nr_synodes_received =
8438 p.requested_synode_app_data.synode_app_data_array_len;
8439 ✗ G_TRACE(
8440 "xcom_client_get_synode_app_data: Got %u synode payloads, we "
8441 "asked "
8442 "for %u.",
8443 nr_synodes_received, nr_synodes_requested);
8444
8445 /* This should always be TRUE.
8446 * But rather than asserting it, let's treat an unexpected number of
8447 * synode payloads in the reply as a failure. */
8448 ✗ if (nr_synodes_received == nr_synodes_requested) {
8449 /* Move (as in C++ move semantics) into reply */
8450 ✗ synode_app_data_array_move(reply, &p.requested_synode_app_data);
8451 ✗ result = success;
8452 }
8453 ✗ break;
8454 }
8455 }
8456 }
8457
8458
1/2
✓ Branch 0 taken 1 times.
✗ Branch 1 not taken.
 1 xdr_free((xdrproc_t)xdr_pax_msg, (char *)&p);
8459
1/2
✓ Branch 0 taken 1 times.
✗ Branch 1 not taken.
 1 xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8460
8461 1 return result;
8462 }
8463
8464 #ifdef NOTDEF
8465 /* Not completely implemented, need to be handled properly
8466 when received as a client message in dispatch_op.
8467 Should have separate opcode from normal add/remove,
8468 like force config_type */
8469 int xcom_client_force_add_node(connection_descriptor *fd, node_list *nl,
8470 uint32_t group_id) {
8471 if (!fd) return 0;
8472 return xcom_send_cfg_wait(fd, nl, group_id, add_node_type, 1);
8473 }
8474
8475 int xcom_client_force_remove_node(connection_descriptor *fd, node_list *nl,
8476 uint32_t group_id) {
8477 if (!fd) return 0;
8478 return xcom_send_cfg_wait(fd, nl, group_id, remove_node_type, 1);
8479 }
8480 #endif
8481
8482 /* purecov: begin deadcode */
8483 ✗ int xcom_client_enable_arbitrator(connection_descriptor *fd) {
8484 ✗ if (!fd) return 0;
8485 app_data a;
8486 ✗ int retval = 0;
8487 ✗ init_app_data(&a);
8488 ✗ a.body.c_t = enable_arbitrator;
8489 ✗ retval = xcom_send_app_wait(fd, &a, 0, nullptr);
8490 ✗ xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8491 ✗ return retval;
8492 }
8493 /* purecov: end */
8494
8495 /* purecov: begin deadcode */
8496 ✗ int xcom_client_disable_arbitrator(connection_descriptor *fd) {
8497 ✗ if (!fd) return 0;
8498 app_data a;
8499 ✗ int retval = 0;
8500 ✗ init_app_data(&a);
8501 ✗ a.body.c_t = disable_arbitrator;
8502 ✗ retval = xcom_send_app_wait(fd, &a, 0, nullptr);
8503 ✗ xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8504 ✗ return retval;
8505 }
8506 /* purecov: end */
8507
8508 /* purecov: begin deadcode */
8509 ✗ int xcom_client_set_cache_limit(connection_descriptor *fd,
8510 uint64_t cache_limit) {
8511 ✗ if (!fd) return 0;
8512 app_data a;
8513 ✗ int retval = 0;
8514 ✗ init_app_data(&a);
8515 ✗ a.body.c_t = set_cache_limit;
8516 ✗ a.body.app_u_u.cache_limit = cache_limit;
8517 ✗ retval = xcom_send_app_wait(fd, &a, 0, nullptr);
8518 ✗ xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8519 ✗ return retval;
8520 }
8521 /* purecov: end */
8522
8523 ✗ int xcom_client_convert_into_local_server(connection_descriptor *const fd) {
8524 ✗ if (!fd) return 0;
8525 app_data a;
8526 ✗ int retval = 0;
8527 ✗ retval = xcom_send_app_wait(fd, init_convert_into_local_server_msg(&a), 0,
8528 nullptr);
8529 ✗ xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8530 ✗ return retval;
8531 }
8532
8533 /* Set max number of leaders */
8534 57 void init_set_max_leaders(uint32_t group_id, app_data *a, node_no max_leaders) {
8535 57 init_app_data(a);
8536 57 a->app_key.group_id = a->group_id = group_id;
8537 57 a->body.c_t = set_max_leaders;
8538 57 a->body.app_u_u.max_leaders = max_leaders;
8539 57 }
8540
8541 /* Set max number of leaders */
8542 4 int xcom_client_set_max_leaders(connection_descriptor *fd, node_no max_leaders,
8543 uint32_t group_id) {
8544
1/2
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
 4 if (!fd) return 0;
8545 app_data a;
8546 ✗ int retval = 0;
8547 ✗ init_set_max_leaders(group_id, &a, max_leaders);
8548 ✗ retval = xcom_send_app_wait(fd, &a, 0, nullptr);
8549 ✗ xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8550 ✗ return retval;
8551 }
8552
8553 55 leader_array new_leader_array(u_int n, char const *names[]) {
8554
1/2
✓ Branch 0 taken 55 times.
✗ Branch 1 not taken.
 55 leader_array leaders = alloc_leader_array(n);
8555
2/2
✓ Branch 0 taken 31 times.
✓ Branch 1 taken 55 times.
 86 for (u_int i = 0; i < n; i++) {
8556 31 leaders.leader_array_val[i].address = strdup(names[i]);
8557 }
8558 55 return leaders;
8559 }
8560
8561 /* Set new set of active leaders. Does not deallocate leaders. */
8562 55 void init_set_leaders(uint32_t group_id, app_data *a,
8563 leader_array const leaders) {
8564 55 init_app_data(a);
8565 55 a->app_key.group_id = a->group_id = group_id;
8566 55 a->body.c_t = set_leaders_type;
8567 // We could have avoided this copy, but having leaders as const
8568 // makes it easier to reason about sharing
8569 55 a->body.app_u_u.leaders = clone_leader_array(leaders);
8570 55 }
8571
8572 /* Set new set of active leaders. */
8573 4 void init_set_leaders(uint32_t group_id, app_data *a, u_int n,
8574 char const *names[]) {
8575
1/2
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
 4 leader_array leaders = new_leader_array(n, names);
8576
1/2
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
 4 init_set_leaders(group_id, a, leaders);
8577 // leaders have been copied, so deallocate
8578
1/2
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
 4 xdr_free((xdrproc_t)xdr_leader_array, (char *)(&leaders));
8579 4 }
8580
8581 51 void init_set_leaders(uint32_t group_id, app_data *leader_app,
8582 leader_array const leaders, app_data *max_app,
8583 node_no max_leaders) {
8584 51 init_set_leaders(group_id, leader_app, leaders);
8585 51 init_set_max_leaders(group_id, max_app, max_leaders);
8586 51 leader_app->next = max_app;
8587 51 }
8588
8589 51 void init_set_leaders(uint32_t group_id, app_data *leader_app, u_int n,
8590 char const *names[], app_data *max_app,
8591 node_no max_leaders) {
8592
1/2
✓ Branch 0 taken 51 times.
✗ Branch 1 not taken.
 51 leader_array leaders = new_leader_array(n, names);
8593
1/2
✓ Branch 0 taken 51 times.
✗ Branch 1 not taken.
 51 init_set_leaders(group_id, leader_app, leaders, max_app, max_leaders);
8594 // leaders have been copied, so deallocate
8595
1/2
✓ Branch 0 taken 51 times.
✗ Branch 1 not taken.
 51 xdr_free((xdrproc_t)xdr_leader_array, (char *)(&leaders));
8596 51 }
8597
8598 /* Set new set of active leaders. */
8599 5 int xcom_client_set_leaders(connection_descriptor *fd, u_int n,
8600 char const *names[], uint32_t group_id) {
8601
1/2
✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
 5 if (!fd) return 0;
8602 app_data a;
8603 ✗ init_set_leaders(group_id, &a, n, names);
8604 ✗ int retval = xcom_send_app_wait(fd, &a, 0, nullptr);
8605 ✗ xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8606 ✗ return retval;
8607 }
8608
8609 std::unique_ptr<Network_provider_management_interface>
8610 58717 get_network_management_interface() {
8611 58717 return std::make_unique<Network_Management_Interface>();
8612 }
8613
8614 std::unique_ptr<Network_provider_operations_interface>
8615 2355 get_network_operations_interface() {
8616 2355 return std::make_unique<Network_Management_Interface>();
8617 }
8618
8619 /* Set new set of active leaders and number of leaders. */
8620 ✗ int xcom_client_set_leaders(connection_descriptor *fd, u_int n,
8621 char const *names[], node_no max_leaders,
8622 uint32_t group_id) {
8623 ✗ if (!fd) return 0;
8624 app_data leader_app;
8625 app_data max_app;
8626 ✗ int retval = 0;
8627 ✗ init_set_leaders(group_id, &leader_app, n, names, &max_app, max_leaders);
8628 ✗ retval = xcom_send_app_wait(fd, &leader_app, 0, nullptr);
8629 // leader_app and max_app have been linked, so unlink
8630 // to avoid deallocating the stack objects.
8631 ✗ leader_app.next = nullptr;
8632 ✗ max_app.next = nullptr;
8633 ✗ xdr_free((xdrproc_t)xdr_app_data, (char *)&leader_app);
8634 ✗ xdr_free((xdrproc_t)xdr_app_data, (char *)&max_app);
8635 ✗ return retval;
8636 }
8637
8638 ✗ int xcom_client_get_leaders(connection_descriptor *fd, uint32_t group_id,
8639 leader_info_data *leaders) {
8640 ✗ if (!fd) return 0;
8641 pax_msg p;
8642 app_data a;
8643 ✗ int result = 0;
8644
8645 ✗ memset(&p, 0, sizeof(p));
8646
8647 ✗ xcom_send_app_wait_result res = xcom_send_app_wait_and_get(
8648 fd, init_get_msg(&a, group_id, get_leaders_type), 0, &p, nullptr);
8649 ✗ result = xcom_check_reply(res);
8650 ✗ if (result) {
8651 // Steal the returned data
8652 ✗ *leaders = steal_leader_info_data(p.rd->reply_data_u.leaders);
8653 }
8654
8655 ✗ xdr_free((xdrproc_t)xdr_pax_msg, (char *)&p);
8656 ✗ xdr_free((xdrproc_t)xdr_app_data, (char *)&a);
8657
8658 ✗ return result;
8659 }
8660
8661 #if 0
8662 /* Called when leader changes */
8663 static xcom_election_cb election_cb [[maybe_unused]] = NULL;
8664
8665 void set_xcom_election_cb(xcom_election_cb x) { election_cb = x; }
8666 #endif
8667
8668 // The timer code and the associated Paxos FSM stuff is only used for
8669 // tracking/debugging Paxos state transitions at the moment, but I believe the
8670 // FSM is correct, and if used for actually handling the incoming messages,
8671 // would make the code simpler, and easier to understand and reason about by
8672 // making lots of tests scattered around in the code unnecessary. I have had
8673 // this on the backburner for a long time, and used the single writer worklog
8674 // to test it, but did not dare to actually replace the existing logic in the
8675 // scope of this worklog.Take it as a hint for future improvement of the code
8676 // base...
8677
8678 // The time queue as configured now will allow up to 10 seconds delay with
8679 // TICK_PERIOD (0.01) seconds granularity. All machines which map to the same
8680 // time slot will wake up simultaneously. The complexity when inserting or
8681 // removing a pax_machine is O(1), but this is somewhat offset by the need to
8682 // advance the current tick for every TICK_PERIOD. Not a problem in practice,
8683 // and the code is dead simple.
8684
8685 /* Max number of ticks before wrapping. With 10 ms per step, this will give a
8686 * max delay of 10 seconds, which is plenty for the Paxos timers */
8687
8688 enum { paxos_timer_range = 1000 };
8689 /* Ten milliseconds granularity is sufficient */
8690 #define TICK_PERIOD 0.01
8691
8692 /* The index into the time queue */
8693 static unsigned int current_tick = 0;
8694
8695 /* The time queue is an array of timers. Each timer is the head of
8696 a possibly empty list of timers */
8697
8698 static linkage time_queue[paxos_timer_range];
8699
8700 2385 static void init_time_queue() {
8701 unsigned int i;
8702
2/2
✓ Branch 0 taken 2385000 times.
✓ Branch 1 taken 2385 times.
 2387385 for (i = 0; i < paxos_timer_range; i++) {
8703 2385000 link_init(&time_queue[i], TYPE_HASH("time_queue"));
8704 }
8705 2385 }
8706
8707 /* Put pax_machine into the time queue at the correct place */
8708 202714 static void paxos_twait(pax_machine *p, unsigned int t) {
8709 /* Guard aginst 0 delay, which would become max delay */
8710
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 202714 times.
 202714 if (0 == t) t = 1;
8711 202714 unsigned int pos = (current_tick + t) % paxos_timer_range;
8712 202714 link_into(&p->watchdog, &time_queue[pos]);
8713
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 202714 times.
 202714 assert(!link_empty(&time_queue[pos]));
8714 202714 }
8715
8716 /* Remove pax_machine from timer queue */
8717 534031 static void paxos_twait_cancel(pax_machine *p) { link_out(&p->watchdog); }
8718
8719 /* Wake all pax_machines waiting at time slot t */
8720 5582887 static void paxos_wakeup(unsigned int t) {
8721 5582887 linkage *head = &time_queue[t];
8722 linkage *p;
8723 5582887 if (!link_empty(head)) {
8724 IFDBG(D_CONS, FN; NUMEXP(t); NUMEXP(link_empty(head)));
8725 }
8726
2/2
✓ Branch 0 taken 2397 times.
✓ Branch 1 taken 5582887 times.
 5585284 while (!link_empty(head)) {
8727 2397 p = link_first(head);
8728 2397 paxos_timeout(container_of(p, pax_machine, watchdog));
8729 2397 link_out(p);
8730 }
8731 5582887 }
8732
8733 /* Advance current_tick to next slot and wake all pax_machines there */
8734 5582887 static void paxos_timer_advance() {
8735 5582887 current_tick = (current_tick + 1) % paxos_timer_range;
8736 5582887 paxos_wakeup(current_tick);
8737 5582887 }
8738
8739 /* Fire any expired timer for a Paxos machine */
8740 5587642 static int paxos_timer_task(task_arg arg [[maybe_unused]]) {
8741 DECL_ENV
8742 double start;
8743 2385 ENV_INIT
8744 2385 END_ENV_INIT
8745 END_ENV;
8746
6/9
✓ Branch 0 taken 2385 times.
✓ Branch 1 taken 5585257 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✓ Branch 4 taken 2385 times.
✓ Branch 5 taken 2385 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 8 times.
✓ Branch 8 taken 2377 times.
 5587642 TASK_BEGIN
8747 2377 ep->start = task_now();
8748
1/2
✓ Branch 0 taken 5585264 times.
✗ Branch 1 not taken.
 5585264 while (!xcom_shutdown) {
8749 5585264 ep->start += TICK_PERIOD;
8750
4/6
✗ Branch 0 not taken.
✓ Branch 1 taken 5585257 times.
✓ Branch 2 taken 5585257 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2370 times.
✓ Branch 5 taken 5582887 times.
 11170521 TASK_DELAY_UNTIL(ep->start);
8751 5582887 paxos_timer_advance();
8752 }
8753 ✗ FINALLY
8754 IFDBG(D_CONS, FN; STRLIT(" shutdown "));
8755
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2378 times.
 2378 TASK_END;
8756 }
8757
8758 /* The state functions/thunks */
8759 static int paxos_fsm_p1_master_enter(pax_machine *paxos, site_def const *site,
8760 paxos_event event, pax_msg *mess);
8761
8762 static int paxos_fsm_p1_master_wait(pax_machine *paxos, site_def const *site,
8763 paxos_event event, pax_msg *mess);
8764
8765 static int paxos_fsm_p2_master_enter(pax_machine *paxos, site_def const *site,
8766 paxos_event event, pax_msg *mess);
8767
8768 static int paxos_fsm_p2_master_wait(pax_machine *paxos, site_def const *site,
8769 paxos_event event, pax_msg *mess);
8770
8771 static int paxos_fsm_p2_slave_enter(pax_machine *paxos, site_def const *site,
8772 paxos_event event, pax_msg *mess);
8773
8774 static int paxos_fsm_p2_slave_wait(pax_machine *paxos, site_def const *site,
8775 paxos_event event, pax_msg *mess);
8776
8777 static int paxos_fsm_p3_master_wait(pax_machine *paxos, site_def const *site,
8778 paxos_event event, pax_msg *mess);
8779
8780 static int paxos_fsm_p3_slave_enter(pax_machine *paxos, site_def const *site,
8781 paxos_event event, pax_msg *mess);
8782
8783 static int paxos_fsm_p3_slave_wait(pax_machine *paxos, site_def const *site,
8784 paxos_event event, pax_msg *mess);
8785
8786 static int paxos_fsm_finished(pax_machine *paxos, site_def const *site,
8787 paxos_event event, pax_msg *mess);
8788
8789 typedef void (*paxos_state_action)(pax_machine *paxos, site_def const *site,
8790 pax_msg *mess);
8791
8792 7768 static int accept_new_prepare(pax_machine *paxos, pax_msg *mess) {
8793
1/2
✓ Branch 0 taken 7768 times.
✗ Branch 1 not taken.
 15536 return noop_match(paxos, mess) ||
8794
2/2
✓ Branch 0 taken 7732 times.
✓ Branch 1 taken 36 times.
 15536 gt_ballot(mess->proposal, paxos->acceptor.promise);
8795 }
8796
8797 10339 static int accept_new_accept(pax_machine *paxos, pax_msg *mess) {
8798
2/2
✓ Branch 0 taken 10140 times.
✓ Branch 1 taken 199 times.
 20479 return noop_match(paxos, mess) ||
8799
1/2
✓ Branch 0 taken 10140 times.
✗ Branch 1 not taken.
 20479 !gt_ballot(paxos->acceptor.promise, mess->proposal);
8800 }
8801
8802 202714 static int own_message(pax_msg *mess, site_def const *site) {
8803 202714 return is_local_node(mess->from, site);
8804 }
8805
8806 // Default paxos timeout in ticks
8807 // Change this if the FSM is used for anything else than debugging
8808 unsigned int constexpr const paxos_default_timeout = 100;
8809
8810 /* You are in a maze of little twisting functions ... */
8811
8812 24390 static void action_paxos_prepare(pax_machine *paxos, site_def const *site,
8813 pax_msg *mess) {
8814
2/2
✓ Branch 0 taken 14525 times.
✓ Branch 1 taken 9865 times.
 24390 if (own_message(mess, site)) {
8815 /* Wait for ack_prepare */
8816 14525 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p1_master_wait);
8817 } else {
8818 /* Wait for accept */
8819 9865 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p2_slave_enter);
8820 }
8821 24390 paxos_twait(paxos, paxos_default_timeout);
8822 24390 }
8823
8824 160253 static void action_paxos_accept(pax_machine *paxos, site_def const *site,
8825 pax_msg *mess) {
8826
2/2
✓ Branch 0 taken 82296 times.
✓ Branch 1 taken 77957 times.
 160253 if (own_message(mess, site)) {
8827 /* Wait for ack_accept */
8828 82296 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p2_master_wait);
8829 } else {
8830 /* Wait for learn */
8831 77957 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p3_slave_enter);
8832 }
8833 160253 paxos_twait(paxos, paxos_default_timeout);
8834 160253 }
8835
8836 534031 static void action_paxos_learn(pax_machine *paxos, site_def const *site,
8837 pax_msg *mess) {
8838 (void)site;
8839 (void)mess;
8840 /* We are finished */
8841 534031 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_finished);
8842 534031 paxos_twait_cancel(paxos);
8843 534031 }
8844
8845 ✗ static void action_paxos_start(pax_machine *paxos, site_def const *site,
8846 pax_msg *mess) {
8847 (void)site;
8848 (void)mess;
8849 /* Find value of this instance */
8850 ✗ SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p1_master_enter);
8851 ✗ paxos_twait(paxos, paxos_default_timeout);
8852 }
8853
8854 7768 static void action_new_prepare(pax_machine *paxos, site_def const *site,
8855 pax_msg *mess) {
8856 (void)site;
8857
2/2
✓ Branch 0 taken 7732 times.
✓ Branch 1 taken 36 times.
 7768 if (accept_new_prepare(paxos, mess)) {
8858 /* Wait for accept */
8859
2/2
✓ Branch 0 taken 7392 times.
✓ Branch 1 taken 340 times.
 7732 if (own_message(mess, site)) {
8860 /* Wait for ack_prepare */
8861 7392 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p1_master_enter);
8862 } else {
8863 /* Wait for accept */
8864 340 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p2_slave_enter);
8865 }
8866 7732 paxos_twait(paxos, paxos_default_timeout);
8867 }
8868 7768 }
8869
8870 23533 static void action_ack_prepare(pax_machine *paxos, site_def const *site,
8871 pax_msg *mess) {
8872 (void)mess;
8873
2/2
✓ Branch 0 taken 9723 times.
✓ Branch 1 taken 13810 times.
 23533 if (check_propose(site, paxos)) {
8874 /* Wait for accept */
8875 9723 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p2_master_enter);
8876 }
8877 23533 }
8878
8879 10339 static void action_new_accept(pax_machine *paxos, site_def const *site,
8880 pax_msg *mess) {
8881
1/2
✓ Branch 0 taken 10339 times.
✗ Branch 1 not taken.
 10339 if (accept_new_accept(paxos, mess)) {
8882 /* Wait for accept */
8883
2/2
✓ Branch 0 taken 498 times.
✓ Branch 1 taken 9841 times.
 10339 if (own_message(mess, site)) {
8884 /* Wait for ack_accept */
8885 498 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p2_master_enter);
8886 } else {
8887 /* Wait for learn */
8888 9841 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p3_slave_enter);
8889 }
8890 10339 paxos_twait(paxos, paxos_default_timeout);
8891 }
8892 10339 }
8893
8894 163235 static void action_ack_accept(pax_machine *paxos, site_def const *site,
8895 pax_msg *mess) {
8896 (void)mess;
8897
2/2
✓ Branch 0 taken 91228 times.
✓ Branch 1 taken 72007 times.
 163235 if (learn_ok(site, paxos)) {
8898 /* Wait for learn message */
8899 91228 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p3_master_wait);
8900 }
8901 163235 }
8902
8903 6801 static void action_ignorant(pax_machine *paxos, site_def const *site,
8904 pax_msg *mess) {
8905 (void)paxos;
8906 (void)site;
8907 (void)mess;
8908 6801 }
8909
8910 /* Dispatch tables for each state */
8911 paxos_state_action p1_idle_vtbl[last_p_event] = {action_paxos_prepare,
8912 nullptr,
8913 action_paxos_accept,
8914 nullptr,
8915 action_paxos_learn,
8916 action_paxos_start,
8917 nullptr};
8918 paxos_state_action p1_master_enter_vtbl[last_p_event] = {action_new_prepare,
8919 action_ack_prepare,
8920 action_new_accept,
8921 nullptr,
8922 action_paxos_learn,
8923 nullptr,
8924 nullptr};
8925 paxos_state_action p1_master_wait_vtbl[last_p_event] = {action_new_prepare,
8926 action_ack_prepare,
8927 action_new_accept,
8928 nullptr,
8929 action_paxos_learn,
8930 nullptr,
8931 nullptr};
8932 paxos_state_action p2_master_enter_vtbl[last_p_event] = {action_new_accept,
8933 nullptr,
8934 action_new_accept,
8935 action_ack_accept,
8936 action_paxos_learn,
8937 nullptr,
8938 nullptr};
8939 paxos_state_action p2_master_wait_vtbl[last_p_event] = {action_new_prepare,
8940 nullptr,
8941 action_new_accept,
8942 action_ack_accept,
8943 action_paxos_learn,
8944 nullptr,
8945 nullptr};
8946 paxos_state_action p2_slave_wait_vtbl[last_p_event] = {action_new_prepare,
8947 nullptr,
8948 action_new_accept,
8949 nullptr,
8950 action_paxos_learn,
8951 nullptr,
8952 nullptr};
8953 paxos_state_action p3_master_wait_vtbl[last_p_event] = {action_new_prepare,
8954 nullptr,
8955 action_new_accept,
8956 nullptr,
8957 action_paxos_learn,
8958 nullptr,
8959 nullptr};
8960 paxos_state_action p3_slave_wait_vtbl[last_p_event] = {action_new_prepare,
8961 nullptr,
8962 action_new_accept,
8963 nullptr,
8964 action_paxos_learn,
8965 nullptr,
8966 nullptr};
8967 paxos_state_action p_finished_vtbl[last_p_event] = {
8968 action_ignorant, nullptr, action_ignorant, nullptr,
8969 nullptr, nullptr, nullptr};
8970
8971 1103421 static inline void dispatch_p_event(paxos_state_action *vtbl,
8972 pax_machine *paxos, site_def const *site,
8973 paxos_event event, pax_msg *mess) {
8974
2/2
✓ Branch 0 taken 930350 times.
✓ Branch 1 taken 173071 times.
 1103421 if (vtbl[event]) {
8975 930350 vtbl[event](paxos, site, mess);
8976 }
8977 1103421 }
8978
8979 /* init state */
8980 534243 int paxos_fsm_idle(pax_machine *paxos, site_def const *site, paxos_event event,
8981 pax_msg *mess) {
8982 IFDBG(D_CONS, FN;);
8983 534243 dispatch_p_event(p1_idle_vtbl, paxos, site, event, mess);
8984 534243 return 0;
8985 }
8986
8987 /* Phase 1 master enter */
8988 7392 static int paxos_fsm_p1_master_enter(pax_machine *paxos, site_def const *site,
8989 paxos_event event, pax_msg *mess) {
8990 (void)site;
8991 (void)event;
8992 (void)mess;
8993 IFDBG(D_CONS, FN;);
8994 /* Send prepare and start timer */
8995 7392 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p1_master_wait);
8996 7392 return 0;
8997 }
8998
8999 /* Phase 1 master wait */
9000 35575 static int paxos_fsm_p1_master_wait(pax_machine *paxos, site_def const *site,
9001 paxos_event event, pax_msg *mess) {
9002 IFDBG(D_CONS, FN;);
9003 35575 dispatch_p_event(p1_master_wait_vtbl, paxos, site, event, mess);
9004 35575 return 0;
9005 }
9006
9007 /* Phase 2 master enter */
9008 10221 static int paxos_fsm_p2_master_enter(pax_machine *paxos, site_def const *site,
9009 paxos_event event, pax_msg *mess) {
9010 (void)site;
9011 (void)event;
9012 (void)mess;
9013 IFDBG(D_CONS, FN;);
9014 /* Send prepare and start timer */
9015 10221 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p2_master_wait);
9016 10221 return 0;
9017 }
9018
9019 /* Phase 2 master wait */
9020 164537 static int paxos_fsm_p2_master_wait(pax_machine *paxos, site_def const *site,
9021 paxos_event event, pax_msg *mess) {
9022 IFDBG(D_CONS, FN;);
9023 164537 dispatch_p_event(p2_master_wait_vtbl, paxos, site, event, mess);
9024 164537 return 0;
9025 }
9026
9027 /* Phase 2 slave enter */
9028 10195 static int paxos_fsm_p2_slave_enter(pax_machine *paxos, site_def const *site,
9029 paxos_event event, pax_msg *mess) {
9030 (void)site;
9031 (void)event;
9032 (void)mess;
9033 IFDBG(D_CONS, FN;);
9034 /* Start timer */
9035 10195 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p2_slave_wait);
9036 10195 return 1;
9037 }
9038
9039 /* Phase 2 slave wait */
9040 10196 static int paxos_fsm_p2_slave_wait(pax_machine *paxos, site_def const *site,
9041 paxos_event event, pax_msg *mess) {
9042 IFDBG(D_CONS, FN;);
9043 10196 dispatch_p_event(p2_slave_wait_vtbl, paxos, site, event, mess);
9044 10196 return 0;
9045 }
9046
9047 /* Phase 3 master wait */
9048 97520 static int paxos_fsm_p3_master_wait(pax_machine *paxos, site_def const *site,
9049 paxos_event event, pax_msg *mess) {
9050 IFDBG(D_CONS, FN;);
9051 97520 dispatch_p_event(p3_master_wait_vtbl, paxos, site, event, mess);
9052 97520 return 0;
9053 }
9054
9055 /* Phase 3 slave enter */
9056 87798 static int paxos_fsm_p3_slave_enter(pax_machine *paxos, site_def const *site,
9057 paxos_event event, pax_msg *mess) {
9058 (void)site;
9059 (void)event;
9060 (void)mess;
9061 IFDBG(D_CONS, FN;);
9062 /* Start timer */
9063 87798 SET_PAXOS_FSM_STATE(paxos, paxos_fsm_p3_slave_wait);
9064 87798 return 1;
9065 }
9066
9067 /* Phase 3 slave wait */
9068 87798 static int paxos_fsm_p3_slave_wait(pax_machine *paxos, site_def const *site,
9069 paxos_event event, pax_msg *mess) {
9070 IFDBG(D_CONS, FN;);
9071 87798 dispatch_p_event(p3_slave_wait_vtbl, paxos, site, event, mess);
9072 87798 return 0;
9073 }
9074
9075 /* Finished */
9076 173552 static int paxos_fsm_finished(pax_machine *paxos, site_def const *site,
9077 paxos_event event, pax_msg *mess) {
9078 IFDBG(D_CONS, FN;);
9079 173552 dispatch_p_event(p_finished_vtbl, paxos, site, event, mess);
9080 173552 return 0;
9081 }
9082
9083 #define X(b) #b
9084 const char *paxos_event_name[] = {p_events};
9085 #undef X
9086
9087 /* Trampoline which loops calling thunks pointed to by paxos->state.state_fp
9088 * until 0 is returned. */
9089 1121034 static void paxos_fsm(pax_machine *paxos, site_def const *site,
9090 paxos_event event, pax_msg *mess) {
9091 /* Crank the state machine until it stops */
9092 IFDBG(D_CONS, FN; PTREXP(paxos); SYCEXP(paxos->synode);
9093 BALCEXP(mess->proposal); STRLIT(paxos->state.state_name); STRLIT(" : ");
9094 STRLIT(paxos_event_name[event]));
9095
2/2
✓ Branch 0 taken 97993 times.
✓ Branch 1 taken 1121034 times.
 1219027 while (paxos->state.state_fp(paxos, site, event, mess)) {
9096 IFDBG(D_CONS, FN; PTREXP(paxos); SYCEXP(paxos->synode);
9097 BALCEXP(mess->proposal); STRLIT(paxos->state.state_name);
9098 STRLIT(" : "); STRLIT(paxos_event_name[event]));
9099 }
9100 1121034 }
9101